Changes for MESSES

A few little changes and bug fixes that came out of updating the messes package.

Author: ptth222

docs/todo.rst

@@ -8,3 +8,24 @@ Hunter also wanted to consider adding things like the _factors properties into t
 
 Think about adding an "UNASSIGNED" data block for the datasets we found that have a results_file instead of having the data in the mwTab file.
 Pretty sure most of these if not all are all unnassigned data where there are basically bins and no metabolite assinments.
+
+
+Had some new undocumented requirements pop up when submitting the Helsley data. The following is an email from Eoin about it:
+Hi Travis,
+You should have received an automated email regarding the completion of your submission.
+There were a couple of problems with the mwtab files. Our submission system requires a single common study design block for all analyses that have the same set of headings for all samples (even if some of them are blank) Headings like Sex and Resected tissue type have to include all samples. 
+Also, the MW needs a mandatory column which indicates the source of each sample-I added one and designated Liver as the source of each patient sample. The “Show all samples link” in the main page (https://dev.metabolomicsworkbench.org:22222/data/subject_fetch.php?STUDY_ID=ST004733&STUDY_TYPE=MS&RESULT_TYPE=1&Access=IloQ2417) shows the proper layout.
+The other issue was that there were a large number of unassigned annotations (m/z_rt features) that may not be added to an mwtab file. These are submitted as tab-delimited text files and saved with the raw data. I pulled these out of the mwtab files and saved them, one for each analysis:
+ST004733_AN008002_Results.txt (124.1K)
+ST004733_AN008003_Results.txt (249.1K)
+ST004733_AN008004_Results.txt (135K)
+ST004733_AN008005_Results.txt (254.2K)
+ST004733_AN008006_Results.txt (109.5K)
+The can be seen  under the “Download raw/supplementary data” on the main page.
+
+Need to address these. New check in validation for all SSF to have the same factors. 
+Must have a new required factor "Sample source" for every sample. It might not have to have that name, but that is 
+the name most common in the latest deposited datasets.
+Add check in validation that Metabolite names aren't just numbers or rt_mz or something like that. 
+Those now have to be submitted as results files.
+

src/mwtab/cli.py

@@ -266,7 +266,7 @@ def cli(cmdargs):
     optional_to_path = cmdargs.get('--to-path')
     optional_output_item = cmdargs.get('--output-item')
     required_output_item = cmdargs.get('<output-item>')
-    download_results_files = cmdargs['--results-files']
+    download_results_files = cmdargs.get('--results-files')
 
 
     # mwtab convert ...

src/mwtab/metadata_column_matching.py

@@ -725,7 +725,7 @@ def make_list_regex(element_regex: str, delimiter: str , quoted_elements: bool =
 
     ColumnFinder("inchi",
                  NameMatcher(in_strings = ['inchi'],
-                             not_in_strings = ['key'],),
+                             not_in_strings = ['key', 'base_inchi', 'representative_inchi', 'isotopic_inchi'],),
                  ValueMatcher(values_regex = '(' + CHEAP_INCHI + '|' + BRACKETED_LIST_OF_INCHI + ')',)),
 
     ColumnFinder("smiles",
@@ -782,6 +782,7 @@ def make_list_regex(element_regex: str, delimiter: str , quoted_elements: bool =
                  ValueMatcher(values_regex = '(' + ION + '|' + \
                                              LIST_OF_IONS + '|' + \
                                              LIST_OF_IONS_SPACE + '|' + \
+                                             LIST_OF_IONS_MIXED + '|' + \
                                              NUMS + '|' + \
                                              LIST_OF_NUMS + '|' + \
                                              NUMS + r'(\s*>\s*|\s*<\s*)' + NUMS + '|' + \

src/mwtab/mwtab.py

@@ -776,7 +776,7 @@ def print_subject_sample_factors(self, section_key, f=sys.stdout, file_format="m
                 # for file missing "Additional sample data" items
                 if len(formatted_items) < 4:
                     line += "\t"
-                print(line, file=f)
+                print(line.replace('\n', ' '), file=f)
 
     def print_block(self, section_key, f=sys.stdout, file_format="mwtab"):
         """Print `mwtab` section into a file or stdout.
@@ -804,13 +804,13 @@ def print_block(self, section_key, f=sys.stdout, file_format="mwtab"):
 
                     results_string = self.prefixes.get(section_key, "") + key + cw * " " + "\t"
                     results_string += self._create_result_file_string(section_key, key, "mwtab")
-                    print(results_string, file=f)
+                    print(results_string.replace('\n', ' '), file=f)
 
                 # prints #MS_METABOLITE_DATA, #NMR_METABOLITE_DATA, or #NMR_BINNED_DATA sections
                 elif key == "Units":
-                    print("{}:UNITS{}\t{}".format(section_key, cw * " ", value), file=f)
+                    print("{}:UNITS{}\t{}".format(section_key, cw * " ", value).replace('\n', ' '), file=f)
                 elif key == "Data":
-                    print("{}_START".format(section_key), file=f)
+                    print("{}_START".format(section_key).replace('\n', ' '), file=f)
 
                     if "METABOLITE" in section_key:
                         # prints "Samples" line at head of data section
@@ -819,7 +819,7 @@ def print_block(self, section_key, f=sys.stdout, file_format="mwtab"):
                             sample_names = self._samples
                         elif self[section_key][key]:
                             sample_names = [k for k in self[section_key][key][0].keys()][1:]
-                        print("\t".join(["Samples"] + sample_names), file=f)
+                        print("\t".join(["Samples"] + sample_names).replace('\n', ' '), file=f)
                         if sample_names:
                             # prints "Factors" line at head of data section
                             if self._factors is not None:
@@ -848,10 +848,10 @@ def print_block(self, section_key, f=sys.stdout, file_format="mwtab"):
                                     break
                                 factors_list.append(factors_dict[k])
                             if factors_list:
-                                print("\t".join(["Factors"] + factors_list), file=f)
+                                print("\t".join(["Factors"] + factors_list).replace('\n', ' '), file=f)
 
                         for k, i in enumerate(self[section_key][key]):
-                            print("\t".join(i.values()), file=f)
+                            print("\t".join(i.values()).replace('\n', ' '), file=f)
 
                     else:  # NMR_BINNED_DATA
                         # Only print if there is data to print.
@@ -860,20 +860,38 @@ def print_block(self, section_key, f=sys.stdout, file_format="mwtab"):
                         elif self[section_key][key]:
                             binned_header = [k for k in self[section_key][key][0].keys()][1:]
 
-                        print("\t".join(["Bin range(ppm)"] + binned_header), file=f)
+                        print("\t".join(["Bin range(ppm)"] + binned_header).replace('\n', ' '), file=f)
 
-                        for i in self[section_key][key]:
-                            print("\t".join(i.values()), file=f)
+                        for i, data_dict in enumerate(self[section_key][key]):
+                            
+                            if 'Bin range(ppm)' in data_dict:
+                                if data_dict['Metabolite'] != data_dict['Bin range(ppm)']:
+                                    print("Warning: The \"Metabolite\" key and \"Bin range(ppm)\" "
+                                          f"key in ['NMR_BINNED_DATA']['Data'][{i}] are different "
+                                          "values. Only the value in \"Bin range(ppm)\" will be written out.")
+                                del self[section_key][key][i]['Metabolite']
+                                print("\t".join(data_dict.values()), file=f)
+                                self[section_key][key][i]['Metabolite'] = self[section_key][key][i]['Bin range(ppm)']
+                            else:
+                                new_dict = self._default_dict_type()
+                                new_dict['Bin range(ppm)'] = data_dict['Metabolite']
+                                del data_dict['Metabolite']
+                                # Note that the update method cannot be used here because it can mess up DuplicatesDict.
+                                for key2, value2 in data_dict.items():
+                                    new_dict[key2] = value2
+                                print("\t".join(new_dict.values()), file=f)
+                                self[section_key][key][i]['Metabolite'] = new_dict['Bin range(ppm)']
+                            
 
-                    print("{}_END".format(section_key), file=f)
+                    print("{}_END".format(section_key).replace('\n', ' '), file=f)
 
                 # prints #METABOLITES section
                 elif key in ("Metabolites", "Extended"):
                     if key == "Metabolites":
                         print("#METABOLITES", file=f)
                         print("METABOLITES_START", file=f)
                     else:
-                        print("EXTENDED_{}_START".format(section_key), file=f)
+                        print("EXTENDED_{}_START".format(section_key).replace('\n', ' '), file=f)
 
                     if key == "Metabolites" and self._metabolite_header is not None:
                         metabolite_header = self._metabolite_header
@@ -883,15 +901,15 @@ def print_block(self, section_key, f=sys.stdout, file_format="mwtab"):
                         metabolite_header = [k for k in self[section_key][key][0].keys()][1:]
                     else:
                         metabolite_header = []
-                    print("\t".join(["metabolite_name"] + metabolite_header), file=f)
+                    print("\t".join(["metabolite_name"] + metabolite_header).replace('\n', ' '), file=f)
 
                     for i in self[section_key][key]:
-                        print("\t".join(i.values()), file=f)
+                        print("\t".join(i.values()).replace('\n', ' '), file=f)
 
                     if key == "Metabolites":
                         print("METABOLITES_END", file=f)
                     else:
-                        print("EXTENDED_{}_END".format(section_key), file=f)
+                        print("EXTENDED_{}_END".format(section_key).replace('\n', ' '), file=f)
 
                 else:
                     # Filenames don't get split.
@@ -908,13 +926,13 @@ def print_block(self, section_key, f=sys.stdout, file_format="mwtab"):
                                 # I fixed this by adding a check to skip filenames, but just in case I also don't 
                                 # let empty lines be printed.
                                 if line:
-                                    print("{}{}{}\t{}".format(self.prefixes.get(section_key, ""), key, cw * " ", " ".join(line)), file=f)
+                                    print("{}{}{}\t{}".format(self.prefixes.get(section_key, ""), key, cw * " ", " ".join(line)).replace('\n', ' '), file=f)
                                 line = [word]
                                 length = len(word)
-                        print("{}{}{}\t{}".format(self.prefixes.get(section_key, ""), key, cw * " ", " ".join(line)),
+                        print("{}{}{}\t{}".format(self.prefixes.get(section_key, ""), key, cw * " ", " ".join(line)).replace('\n', ' '),
                               file=f)
                     else:
-                        print("{}{}{}\t{}".format(self.prefixes.get(section_key, ""), key, cw * " ", value), file=f)
+                        print("{}{}{}\t{}".format(self.prefixes.get(section_key, ""), key, cw * " ", value).replace('\n', ' '), file=f)
 
         # Note that indent cannot be None or json will use a version of the 
         # encoder written in C and DuplicatesDict will not be printed correctly.
@@ -1101,7 +1119,7 @@ def _set_key_order(self):
         Sets the key order to a certain order for better reproducibility and consistency.
         """               
         key_order = \
-            {'METABOLOMICS WORKBENCH': {},
+            {'METABOLOMICS WORKBENCH': {'STUDY_ID': [], 'ANALYSIS_ID': [], 'VERSION': [], 'CREATED_ON': []},
              'PROJECT': {},
              'STUDY': {},
              'SUBJECT': {},

src/mwtab/validator.py

@@ -360,6 +360,10 @@ def validate_metabolites(mwtabfile, data_section_key, mwtabfile_tables):
                         metabolites_errors.append({'message': message, 'tags': ['value'], 'section': data_section_key, 'sub-section': 'Metabolites',
                                                    'ID': '15', 'name': 'Standard Column Name Match'})
             for column_name in column_matches:
+                # Do not want to check columns that look similar if a direct match already exits, for example retention_time and retention_time%units.
+                # A message for retention_time%units would be spurious.
+                if name in columns and column_name.lower() != name:
+                    continue
                 if column_name in columns_to_standard_columns:
                     columns_to_standard_columns[column_name].append(name)
                 else:
@@ -797,7 +801,8 @@ def create_better_error_messages(errors_generator: Iterable[jsonschema.exception
                 container_noun = 'key'
 
             if key_is_required:
-                message = message + f' A legitimate value should be provided for this required {container_noun}.'
+                message = message + f' A legitimate value should be provided for this required {container_noun}.' + \
+                          ' You may have to ignore this error in certain situations. For example, SOLVENT_A for a GCMS experiment.'
             else:
                 message = message + f' Either a legitimate value should be provided for this {container_noun}, or it should be removed altogether.'
         else:
@@ -930,7 +935,10 @@ def validate_table_values(mwtabfile, data_section_key, mwtabfile_tables, na_valu
             # Look for overbalanced values, so if 90% of a column is dominated by a single value print a warning.
             for i, column in enumerate([column for column in data_df.columns if column != 'Metabolite']):
                 temp_column = data_df.loc[:, column].astype(str)
-                value_counts = temp_column.value_counts(dropna=False)
+                # There are fairly sparse data sets where this check will just be spurious if blanks/NAs are included.
+                value_counts = temp_column.value_counts(dropna = True if table_name == 'Data' else False)
+                if table_name == 'Data':
+                    value_counts = value_counts[value_counts.index != '']
                 value_counts = value_counts / value_counts.sum()
                 if any(value_counts > .9) and len(value_counts) > 1:
                     message = (f'Warning: {format_column_name(column, data_df.columns.get_loc(column)+1)} '