Skip to content

Commit cb25ae5

Browse files
ThisCakeIsALieThisCakeIsALie
authored
Add new Reqs2x settings (#313)
Adds a bunch of new misc settings for Reqs2x (subsumes some prior tiny branches): - Control retries of test generation - Control passing of additional test examples for test generation - Control reordering of entries in exported files (for the extension this only really affects tests) - Control compatibility mode for the LLM providers
1 parent 71e0926 commit cb25ae5

File tree

3 files changed

+70
-21
lines changed

3 files changed

+70
-21
lines changed

‎package.json‎

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -471,9 +471,27 @@
471471
"default": false,
472472
"description": "Output debug information for requirement and test generation to the console"
473473
},
474+
"vectorcastTestExplorer.reqs2x.retries": {
475+
"type": "integer",
476+
"order": 6,
477+
"default": 2,
478+
"description": "Number of retries test generation. WARNING: High values (over 3) may lead to increased costs and longer processing times."
479+
},
480+
"vectorcastTestExplorer.reqs2x.reorder": {
481+
"type": "boolean",
482+
"order": 7,
483+
"default": true,
484+
"description": "Reorder test cases exported to file (disable for easier investigation of incremental results)"
485+
},
486+
"vectorcastTestExplorer.reqs2x.noTestExamples": {
487+
"type": "boolean",
488+
"order": 8,
489+
"default": false,
490+
"description": "Do not provide additional test examples to the LLM during test generation"
491+
},
474492
"vectorcastTestExplorer.reqs2x.generationLanguage": {
475493
"type": "string",
476-
"order": 6,
494+
"order": 9,
477495
"default": "en",
478496
"enum": [
479497
"af", "sq", "am", "ar", "hy", "as", "az", "be", "bn", "bs", "bg", "my", "ch", "hr", "cs", "da", "nl", "en", "et", "tl", "fi", "fr", "ka", "de", "gu", "ha", "he", "hi", "hu", "is", "ig", "id", "ga", "it", "jp", "kn", "kk", "km", "ko", "ky", "lo", "lv", "lt", "mk", "ms", "ml", "mt", "mn", "me", "ne", "no", "or", "fa", "pl", "pt", "pa", "ro", "ru", "sr", "si", "sk", "sl", "es", "sw", "sv", "tg", "ta", "te", "th", "tr", "uk", "ur", "uz", "vi", "cy", "xh", "yo", "zu"
@@ -483,113 +501,119 @@
483501
],
484502
"description": "Generation language for requirements and test generation responses"
485503
},
504+
"vectorcastTestExplorer.reqs2x.modelCompatibilityMode": {
505+
"type": "boolean",
506+
"order": 10,
507+
"default": false,
508+
"description": "Enable model compatibility mode (runs Reqs2X tools without structured outputs which enables compatibility with more providers)."
509+
},
486510
"vectorcastTestExplorer.reqs2x.provider": {
487511
"type": "string",
488-
"order": 7,
512+
"order": 11,
489513
"default": "openai",
490514
"enum": ["openai", "azure_openai", "anthropic", "litellm"],
491515
"enumDescriptions": ["OpenAI", "Azure OpenAI", "Anthropic", "LiteLLM"],
492516
"description": "Language model provider for automatic requirements and test generation using Reqs2X"
493517
},
494518
"vectorcastTestExplorer.reqs2x.openai.apiKey": {
495519
"type": "string",
496-
"order": 8,
520+
"order": 12,
497521
"default": "",
498522
"description": "OpenAI API key"
499523
},
500524
"vectorcastTestExplorer.reqs2x.openai.modelName": {
501525
"type": "string",
502-
"order": 9,
526+
"order": 13,
503527
"default": "",
504528
"description": "OpenAI model name"
505529
},
506530
"vectorcastTestExplorer.reqs2x.openai.reasoningModelName": {
507531
"type": "string",
508-
"order": 10,
532+
"order": 14,
509533
"default": "",
510534
"description": "Optional OpenAI reasoning model name used for specialized reasoning subtasks."
511535
},
512536
"vectorcastTestExplorer.reqs2x.openai.baseUrl": {
513537
"type": "string",
514-
"order": 11,
538+
"order": 15,
515539
"default": "",
516540
"description": "OpenAI base URL (optional, set this to use OpenAI-compatible providers such as Ollama, vLLM or Bedrock)"
517541
},
518542
"vectorcastTestExplorer.reqs2x.azure.baseUrl": {
519543
"type": "string",
520-
"order": 12,
544+
"order": 16,
521545
"default": "",
522546
"description": "Azure OpenAI endpoint/base URL, e.g., https://my-example-instance.openai.azure.com"
523547
},
524548
"vectorcastTestExplorer.reqs2x.azure.apiKey": {
525549
"type": "string",
526-
"order": 13,
550+
"order": 17,
527551
"default": "",
528552
"description": "Azure OpenAI API key"
529553
},
530554
"vectorcastTestExplorer.reqs2x.azure.deployment": {
531555
"type": "string",
532-
"order": 14,
556+
"order": 18,
533557
"default": "",
534558
"description": "Azure OpenAI deployment name, e.g., my-custom-gpt-4o-deployment"
535559
},
536560
"vectorcastTestExplorer.reqs2x.azure.modelName": {
537561
"type": "string",
538-
"order": 15,
562+
"order": 19,
539563
"default": "",
540564
"description": "Azure OpenAI model name, e.g., gpt-4o"
541565
},
542566
"vectorcastTestExplorer.reqs2x.azure.reasoningModelName": {
543567
"type": "string",
544-
"order": 16,
568+
"order": 20,
545569
"default": "",
546570
"description": "Optional Azure OpenAI reasoning model name used for specialized reasoning subtasks."
547571
},
548572
"vectorcastTestExplorer.reqs2x.azure.reasoningDeployment": {
549573
"type": "string",
550-
"order": 17,
574+
"order": 21,
551575
"default": "",
552576
"description": "Azure OpenAI reasoning deployment name (required if Azure reasoning model name is provided)."
553577
},
554578
"vectorcastTestExplorer.reqs2x.azure.apiVersion": {
555579
"type": "string",
556-
"order": 18,
580+
"order": 22,
557581
"default": "2024-12-01-preview",
558582
"description": "Azure OpenAI API version"
559583
},
560584
"vectorcastTestExplorer.reqs2x.anthropic.apiKey": {
561585
"type": "string",
562-
"order": 19,
586+
"order": 23,
563587
"default": "",
564588
"description": "Anthropic API key"
565589
},
566590
"vectorcastTestExplorer.reqs2x.anthropic.modelName": {
567591
"type": "string",
568-
"order": 20,
592+
"order": 24,
569593
"default": "",
570594
"description": "Anthropic model name"
571595
},
572596
"vectorcastTestExplorer.reqs2x.anthropic.reasoningModelName": {
573597
"type": "string",
574-
"order": 21,
598+
"order": 25,
575599
"default": "",
576600
"description": "Optional Anthropic reasoning model name used for specialized reasoning subtasks."
577601
},
578602
"vectorcastTestExplorer.reqs2x.litellm.modelName": {
579603
"type": "string",
580-
"order": 22,
604+
"order": 26,
581605
"default": "",
582606
"description": "LiteLLM model name (prefix with provider used, e.g., openai/gpt-4o or azure/o3-mini)"
583607
},
584608
"vectorcastTestExplorer.reqs2x.litellm.reasoningModelName": {
585609
"type": "string",
586-
"order": 23,
610+
"order": 27,
587611
"default": "",
588612
"description": "Optional LiteLLM reasoning model name used for specialized reasoning subtasks."
589613
},
590614
"vectorcastTestExplorer.reqs2x.litellm.providerEnvVars": {
591615
"type": "string",
592-
"order": 24,
616+
"order": 28,
593617
"default": "",
594618
"description": "Environment variables to set when running LiteLLM, e.g., OPENAI_API_KEY=xxxx. Multiple variables can be separated by commas, e.g., OPENAI_API_KEY=xxxx,ANOTHER_ENV_VAR=yyyy. For a list of variables to set for a specific LiteLLM-compatible provider, see https://docs.litellm.ai/docs/providers."
595619
}

‎src/requirements/requirementsOperations.ts‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ export async function generateRequirements(enviroPath: string) {
239239
"generateHighLevelRequirements",
240240
false
241241
);
242+
const reorder = config.get<boolean>("reorder", true);
242243

243244
const commandArgs = [
244245
"-e",
@@ -256,6 +257,10 @@ export async function generateRequirements(enviroPath: string) {
256257
commandArgs.push("--generate-high-level-requirements");
257258
}
258259

260+
if (!reorder) {
261+
commandArgs.push("--no-reorder");
262+
}
263+
259264
// Log the command being executed
260265
const commandString = `${CODE2REQS_EXECUTABLE_PATH} ${commandArgs.join(" ")}`;
261266
logCliOperation(`Executing command: ${commandString}`);
@@ -363,11 +368,25 @@ export async function generateTestsFromRequirements(
363368
}
364369

365370
// Get the decompose setting from configuration
366-
const config = vscode.workspace.getConfiguration("vectorcastTestExplorer");
371+
const config = vscode.workspace.getConfiguration(
372+
"vectorcastTestExplorer.reqs2x"
373+
);
367374
const decomposeRequirements = config.get<boolean>(
368375
"decomposeRequirements",
369376
true
370377
);
378+
379+
const noTestExamples = config.get<boolean>("noTestExamples", false);
380+
const reorder = config.get<boolean>("reorder", true);
381+
382+
const retries = config.get<number>("retries", 2);
383+
if (retries < 1) {
384+
vscode.window.showErrorMessage(
385+
"Retries must be greater than or equal to 1. Please check your settings."
386+
);
387+
return;
388+
}
389+
371390
const enableRequirementKeys =
372391
findRelevantRequirementGateway(enviroPath) !== null;
373392
console.log(decomposeRequirements, enableRequirementKeys);
@@ -380,9 +399,11 @@ export async function generateTestsFromRequirements(
380399
"--export-tst",
381400
tstPath,
382401
"--retries",
383-
"1",
402+
retries.toString(),
384403
"--batched",
385404
...(decomposeRequirements ? [] : ["--no-requirement-decomposition"]),
405+
...(noTestExamples ? ["--no-test-examples"] : []),
406+
...(!reorder ? ["--no-reorder"] : []),
386407
"--allow-partial",
387408
"--json-events",
388409
...(enableRequirementKeys ? ["--requirement-keys"] : []),

‎src/requirements/requirementsUtils.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,10 @@ export async function createProcessEnvironment(): Promise<NodeJS.ProcessEnv> {
548548
processEnv.VCAST_REQS2X_LOG_LEVEL = "debug";
549549
}
550550

551+
if (config.get<boolean>("modelCompatibilityMode", false)) {
552+
processEnv.VCAST_REQS2X_MODEL_COMPATIBILITY_MODE = "1";
553+
}
554+
551555
// Return the constructed environment
552556
return processEnv;
553557
}

0 commit comments

Comments
 (0)