Merge pull request #59 from StepenkoAnatoli/fix/perquery-ceiling

max_pages was capped at 3 by perQuery, on every dispatched collection

Author: StepenkoAnatoli

.github/workflows/collect.yml

@@ -374,10 +374,18 @@ jobs:
             const plan = JSON.parse(fs.readFileSync("research/plan.json", "utf8"));
             plan.topic = topic;
             plan.maxScrapes = Number(process.env.MAX_PAGES);
+            // perQuery is the REAL ceiling on breadth: selectCandidates slices the
+            // results of each query to it, so one seeded query and a default of 3
+            // capped every dispatched run at three candidates however large max_pages
+            // was. Matching it to max_pages is what makes the advertised 1..25 control
+            // mean what it says. Nothing about SPEND changes - maxScrapes still bounds
+            // the fetches, and this only widens the list they are chosen from.
+            plan.perQuery = Number(process.env.MAX_PAGES);
+            plan.limit = Math.max(Number(process.env.MAX_PAGES), plan.limit ?? 8);
             plan.queries = [{ q: topic, why: "the dispatched topic", prefer: [] }];
             plan.urls = [];
             fs.writeFileSync("research/plan.json", `${JSON.stringify(plan, null, 2)}\n`, "utf8");
-            console.log(`plan written: 1 query, maxScrapes ${plan.maxScrapes}`);
+            console.log(`plan written: 1 query, maxScrapes ${plan.maxScrapes}, perQuery ${plan.perQuery}`);
           '
 
       - name: seed the map, spending nothing

.github/workflows/live-collection.yml

@@ -205,6 +205,7 @@ jobs:
             const p = "research/plan.json";
             const plan = JSON.parse(fs.readFileSync(p, "utf8"));
             plan.maxScrapes = Number(process.env.MAX_PAGES);
+            plan.perQuery = Number(process.env.MAX_PAGES);
             fs.writeFileSync(p, JSON.stringify(plan, null, 2));
             console.log("plan written: maxScrapes " + plan.maxScrapes);
           '