Enable listing of zip files in fromfiles for index, gather, search

src/mastiff_manygather.rs

@@ -34,6 +34,18 @@ pub fn mastiff_manygather(
         allow_failed_sigpaths,
     )?;
 
+    eprintln!(
+        "queries_file: {}\nquery_collection.len(): {}",
+        // "\nquery_collection.manifest().internal_locations(): {}",
+        queries_file,
+        query_collection.len(),
+        // query_collection.manifest().internal_locations().cloned().collect::<Vec<String>>().join(", ")
+    );
+
+    query_collection.manifest().internal_locations().for_each(|location|
+        eprintln!("query collection, internal location: {}", location)
+    );
+
     // set up a multi-producer, single-consumer channel.
     let (send, recv) =
         std::sync::mpsc::sync_channel::<BranchwaterGatherResult>(rayon::current_num_threads());

src/python/tests/test_gather.py

@@ -221,7 +221,7 @@ def test_bad_against(runtmp, capfd):
     captured = capfd.readouterr()
     print(captured.err)
 
-    assert "WARNING: could not load sketches from path 'no-exist'" in captured.err
+    assert "WARNING: path 'no-exist' does not exist" in captured.err
     assert "WARNING: 1 search paths failed to load. See error messages above." in captured.err
 
 
@@ -247,7 +247,7 @@ def test_bad_against_2(runtmp, capfd):
     captured = capfd.readouterr()
     print(captured.err)
 
-    assert "Sketch loading error: File is too short, less than five bytes" in captured.err
+    # assert "Sketch loading error: File is too short, less than five bytes" in captured.err
     assert "WARNING: could not load sketches from path" in captured.err
 
     assert "WARNING: 1 search paths failed to load. See error messages above." in captured.err
@@ -307,7 +307,7 @@ def test_against_multisigfile(runtmp, zip_against):
         print(df)
     else:
         print(df)
-        assert len(df) == 1
+        assert len(df) == 3
     # @CTB this is a bug :(. It should load multiple sketches properly!
 
 

src/python/tests/test_index.py

@@ -123,6 +123,32 @@ def test_index_manifest(runtmp, capfd):
     assert 'index is done' in runtmp.last_result.err
 
 
+def test_index_manifest_zip_files(runtmp, capfd):
+    # test index with text file of multiple zip files
+    sig2 = get_test_data('2.fa.sig.gz')
+    sig47 = get_test_data('47.fa.sig.gz')
+    sig63 = get_test_data('63.fa.sig.gz')
+    sigs = [sig2, sig47, sig63]
+    manifests = []
+    for sig in sigs:
+        sig_mf = runtmp.output(os.path.basename(sig) + ".mf.zip")
+        runtmp.sourmash("sig", "cat", sig, "-o", sig_mf)
+        manifests.append(sig_mf)
+
+    # assert False
+    manifests_zips_list = runtmp.output('manifest_zips.txt')
+    make_file_list(manifests_zips_list, manifests)
+
+    output = runtmp.output('out.db')
+    runtmp.sourmash('scripts', 'index', manifests_zips_list,
+                        '-o', output)
+
+    captured = capfd.readouterr()
+    print(captured.err)
+    print(runtmp.last_result.err)
+    assert 'index is done' in runtmp.last_result.err
+
+
 def test_index_bad_siglist_2(runtmp, capfd):
     # test with a bad siglist (containing a missing file)
     against_list = runtmp.output('against.txt')
@@ -140,7 +166,7 @@ def test_index_bad_siglist_2(runtmp, capfd):
 
     captured = capfd.readouterr()
     print(captured.err)
-    assert "WARNING: could not load sketches from path 'no-exist'" in captured.err
+    assert "WARNING: path 'no-exist' does not exist" in captured.err
 
 
 def test_index_empty_siglist(runtmp, capfd):

src/python/tests/test_multigather.py

@@ -396,7 +396,7 @@ def test_missing_query(runtmp, capfd, indexed):
 
     captured = capfd.readouterr()
     print(captured.err)
-    assert "WARNING: could not load sketches from path 'no-exist'" in captured.err
+    assert "WARNING: path 'no-exist' does not exist" in captured.err
     assert "WARNING: 1 query paths failed to load. See error messages above."
 
 
@@ -492,7 +492,7 @@ def test_bad_against(runtmp, capfd):
 
     against_list = runtmp.output('against.txt')
     sig2 = get_test_data('2.fa.sig.gz')
-    make_file_list(against_list, [sig2, "no exist"])
+    make_file_list(against_list, [sig2, "no-exist"])
 
     # should succeed, but with error output.
     runtmp.sourmash('scripts', 'fastmultigather', query_list, against_list,
@@ -501,7 +501,7 @@ def test_bad_against(runtmp, capfd):
     captured = capfd.readouterr()
     print(captured.err)
 
-    assert "WARNING: could not load sketches from path 'no exist'" in captured.err
+    assert "WARNING: path 'no-exist' does not exist" in captured.err
     assert "WARNING: 1 search paths failed to load. See error messages above." in captured.err
 
 
@@ -548,7 +548,7 @@ def test_empty_against(runtmp, capfd):
     captured = capfd.readouterr()
     print(captured.err)
 
-    assert "Sketch loading error: No such file or directory" in captured.err
+    assert "WARNING: path '' does not exist in 'against.txt'" in captured.err
     assert "No search signatures loaded, exiting." in captured.err
 
 

src/python/tests/test_multisearch.py

@@ -312,7 +312,7 @@ def test_bad_query(runtmp, capfd):
     captured = capfd.readouterr()
     print(captured.err)
 
-    assert "WARNING: could not load sketches from path 'no-exist'" in captured.err
+    assert "WARNING: path 'no-exist' does not exist" in captured.err
     assert "WARNING: 1 query paths failed to load. See error messages above." in captured.err
 
 
@@ -416,7 +416,7 @@ def test_bad_against(runtmp, capfd):
     captured = capfd.readouterr()
     print(captured.err)
 
-    assert "WARNING: could not load sketches from path 'no-exist'" in captured.err
+    assert "WARNING: path 'no-exist' does not exist" in captured.err
     assert "WARNING: 1 search paths failed to load. See error messages above." in captured.err
 
 

src/python/tests/test_pairwise.py

@@ -220,7 +220,7 @@ def test_bad_query(runtmp, capfd):
     captured = capfd.readouterr()
     print(captured.err)
 
-    assert "WARNING: could not load sketches from path 'no-exist'" in captured.err
+    assert "WARNING: path 'no-exist' does not exist" in captured.err
     assert "WARNING: 1 analysis paths failed to load. See error messages above." in captured.err
 
 

src/python/tests/test_search.py

@@ -350,7 +350,7 @@ def test_bad_query_2(runtmp, capfd, indexed):
     captured = capfd.readouterr()
     print(captured.err)
 
-    assert "WARNING: could not load sketches from path 'no-exist'" in captured.err
+    assert "WARNING: path 'no-exist' does not exist" in captured.err
     assert "WARNING: 1 query paths failed to load. See error messages above." in captured.err
 
 
@@ -449,7 +449,7 @@ def test_bad_against(runtmp, capfd):
     captured = capfd.readouterr()
     print(captured.err)
 
-    assert "WARNING: could not load sketches from path 'no-exist'" in captured.err
+    assert "WARNING: path 'no-exist' does not exist" in captured.err
     assert "WARNING: 1 search paths failed to load. See error messages above." in captured.err