remove gin

Author: linjianma

recommendation/dlrm_v3/README.md

@@ -1,53 +1,68 @@
-# MLCommons (MLPerf) DLRMv3 Inference Benchmarks
+# MLPerf Inference reference implementation for DLRMv3
 
-## Install generative-recommenders
+## Install dependencies and build loadgen
 
 ```
-cd generative_recommenders/
-pip install -e .
+sh setup.sh
 ```
 
-## Build loadgen
+## Dataset download
 
-```
-cd generative_recommenders/generative_recommenders/dlrm_v3/inference/thirdparty/loadgen/
-CFLAGS="-std=c++14 -O3" python -m pip install .
-```
-
-## Generate synthetic dataset
-
-```
-cd generative_recommenders/dlrm_v3/
-python streaming_synthetic_data.py
-```
+TODO: pending MLPerf system setup
 
 ## Inference benchmark
 
 ```
-cd generative_recommenders/generative_recommenders/dlrm_v3/inference/
-WORLD_SIZE=8 python main.py --dataset streaming-100b
+WORLD_SIZE=8 python main.py --dataset sampled-streaming-100b
 ```
 
-The config file is listed in `dlrm_v3/inference/gin/streaming_100b.gin`.
 `WORLD_SIZE` is the number of GPUs used in the inference benchmark.
 
-To load checkpoint from training, modify `run.model_path` inside the inference
-gin config file. (We will relase the checkpoint soon.)
-
-To achieve the best performance, tune `run.target_qps` and `run.batch_size` in
-the config file.
+```
+usage: main.py [-h] [--dataset {streaming-100b,sampled-streaming-100b}] [--model-path MODEL_PATH] [--scenario-name {SingleStream,MultiStream,Server,Offline}] [--batchsize BATCHSIZE]
+               [--output-trace OUTPUT_TRACE] [--data-producer-threads DATA_PRODUCER_THREADS] [--compute-eval COMPUTE_EVAL] [--find-peak-performance FIND_PEAK_PERFORMANCE]
+               [--dataset-path-prefix DATASET_PATH_PREFIX] [--warmup-ratio WARMUP_RATIO] [--num-queries NUM_QUERIES] [--target-qps TARGET_QPS] [--numpy-rand-seed NUMPY_RAND_SEED]
+               [--sparse-quant SPARSE_QUANT] [--dataset-percentage DATASET_PERCENTAGE]
+
+options:
+  -h, --help            show this help message and exit
+  --dataset {streaming-100b,sampled-streaming-100b}
+                        name of the dataset
+  --model-path MODEL_PATH
+                        path to the model checkpoint. Example: /home/username/ckpts/streaming_100b/89/
+  --scenario-name {SingleStream,MultiStream,Server,Offline}
+                        inference benchmark scenario
+  --batchsize BATCHSIZE
+                        batch size used in the benchmark
+  --output-trace OUTPUT_TRACE
+                        Whether to output trace
+  --data-producer-threads DATA_PRODUCER_THREADS
+                        Number of threads used in data producer
+  --compute-eval COMPUTE_EVAL
+                        If true, will run AccuracyOnly mode and outputs both predictions and labels for accuracy calcuations
+  --find-peak-performance FIND_PEAK_PERFORMANCE
+                        Whether to find peak performance in the benchmark
+  --dataset-path-prefix DATASET_PATH_PREFIX
+                        Prefix to the dataset path. Example: /home/username/
+  --warmup-ratio WARMUP_RATIO
+                        The ratio of the dataset used to warmup SUT
+  --num-queries NUM_QUERIES
+                        Number of queries to run in the benchmark
+  --target-qps TARGET_QPS
+                        Benchmark target QPS. Needs to be tuned for different implementations to balance latency and throughput
+  --numpy-rand-seed NUMPY_RAND_SEED
+                        Numpy random seed
+  --sparse-quant SPARSE_QUANT
+                        Whether to quantize sparse arch
+  --dataset-percentage DATASET_PERCENTAGE
+                        Percentage of the dataset to run in the benchmark
+```
 
 ## Accuracy test
 
 Set `run.compute_eval` will run the accuracy test and dump prediction outputs in
 `mlperf_log_accuracy.json`. To check the accuracy, run
 
 ```
-python accuracy.py -- --path path/to/mlperf_log_accuracy.json
-```
-
-## Run unit tests
-
-```
-python tests/inference_test.py
+python accuracy.py --path path/to/mlperf_log_accuracy.json
 ```

recommendation/dlrm_v3/gin/streaming_100b.gin

@@ -30,8 +30,6 @@
 import time
 from typing import Any, Dict, List, Optional, Union
 
-import gin
-
 # pyre-ignore [21]
 import mlperf_loadgen as lg  # @manual
 import numpy as np
@@ -64,10 +62,6 @@
 
 USER_CONF = f"{os.path.dirname(__file__)}/user.conf"
 
-SUPPORTED_CONFIGS = {
-    "sampled-streaming-100b": "streaming_100b.gin",
-}
-
 
 SCENARIO_MAP = {  # pyre-ignore [5]
     "SingleStream": lg.TestScenario.SingleStream,
@@ -81,7 +75,49 @@ def get_args():  # pyre-ignore [3]
     """Parse commandline."""
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--dataset", default="debug", choices=SUPPORTED_DATASETS, help="dataset"
+        "--dataset", default="sampled-streaming-100b", choices=SUPPORTED_DATASETS, help="name of the dataset"
+    )
+    parser.add_argument(
+        "--model-path", default="", help="path to the model checkpoint. Example: /home/username/ckpts/streaming_100b/89/"
+    )
+    parser.add_argument(
+        "--scenario-name", default="Server", choices={"SingleStream", "MultiStream", "Server", "Offline"}, help="inference benchmark scenario"
+    )
+    parser.add_argument(
+        "--batchsize", default=20, help="batch size used in the benchmark"
+    )
+    parser.add_argument(
+        "--output-trace", default=False, help="Whether to output trace"
+    )
+    parser.add_argument(
+        "--data-producer-threads", default=16, help="Number of threads used in data producer"
+    )
+    parser.add_argument(
+        "--compute-eval", default=False, help="If true, will run AccuracyOnly mode and outputs both predictions and labels for accuracy calcuations"
+    )
+    parser.add_argument(
+        "--find-peak-performance", default=False, help="Whether to find peak performance in the benchmark"
+    )
+    parser.add_argument(
+        "--dataset-path-prefix", default="", help="Prefix to the dataset path. Example: /home/username/"
+    )
+    parser.add_argument(
+        "--warmup-ratio", default=0.1, help="The ratio of the dataset used to warmup SUT"
+    )
+    parser.add_argument(
+        "--num-queries", default=500000, help="Number of queries to run in the benchmark"
+    )
+    parser.add_argument(
+        "--target-qps", default=1500, help="Benchmark target QPS. Needs to be tuned for different implementations to balance latency and throughput"
+    )
+    parser.add_argument(
+        "--numpy-rand-seed", default=123, help="Numpy random seed"
+    )
+    parser.add_argument(
+        "--sparse-quant", default=False, help="Whether to quantize sparse arch"
+    )
+    parser.add_argument(
+        "--dataset-percentage", default=0.001, help="Percentage of the dataset to run in the benchmark"
     )
     args, unknown_args = parser.parse_known_args()
     logger.warning(f"unknown_args: {unknown_args}")
@@ -363,33 +399,24 @@ def get_item_count(self) -> int:
         return self.total_requests
 
 
-@gin.configurable
 def run(
     dataset: str = "debug",
     model_path: str = "",
     scenario_name: str = "Server",
     batchsize: int = 16,
-    out_dir: str = "",
     output_trace: bool = False,
     data_producer_threads: int = 4,
     compute_eval: bool = False,
     find_peak_performance: bool = False,
-    new_path_prefix: str = "",
-    train_split_percentage: float = 0.75,
+    dataset_path_prefix: str = "",
     warmup_ratio: float = 0.1,
-    # below will override mlperf rules compliant settings - don't use for official submission
-    duration: Optional[int] = None,
     target_qps: Optional[int] = None,
-    max_latency: Optional[float] = None,
     num_queries: Optional[int] = None,
-    samples_per_query_multistream: int = 8,
-    max_num_samples: int = -1,
     numpy_rand_seed: int = 123,
-    dev_mode: bool = False,
     sparse_quant: bool = False,
     dataset_percentage: float = 1.0,
 ) -> None:
-    set_dev_mode(dev_mode)
+    set_dev_mode(False)
     if scenario_name not in SCENARIO_MAP:
         raise NotImplementedError("valid scanarios:" + str(list(SCENARIO_MAP.keys())))
     scenario = SCENARIO_MAP[scenario_name]
@@ -408,7 +435,7 @@ def run(
         compute_eval=compute_eval,
     )
     is_streaming: bool = "streaming" in dataset
-    dataset, kwargs = get_dataset(dataset, new_path_prefix)
+    dataset, kwargs = get_dataset(dataset, dataset_path_prefix)
 
     ds: Dataset = dataset(
         hstu_config=hstu_config,
@@ -430,11 +457,6 @@ def run(
         logger.error("{} not found".format(user_conf))
         sys.exit(1)
 
-    if out_dir:
-        output_dir = os.path.abspath(out_dir)
-        os.makedirs(output_dir, exist_ok=True)
-        os.chdir(output_dir)
-
     # warmup
     warmup_ids = list(range(batchsize))
     ds.load_query_samples(warmup_ids)
@@ -453,7 +475,7 @@ def run(
         if not is_streaming
         else ds.get_item_count()
     )
-    train_size: int = round(train_split_percentage * count) if not is_streaming else 0
+    train_size: int = 0
 
     settings = lg.TestSettings()
     settings.FromConfig(user_conf, model_path, scenario_name)
@@ -489,21 +511,10 @@ def flush_queries() -> None:
     if find_peak_performance:
         settings.mode = lg.TestMode.FindPeakPerformance
 
-    if duration:
-        settings.min_duration_ms = duration
-        settings.max_duration_ms = duration
-
     if target_qps:
         settings.server_target_qps = float(target_qps)
         settings.offline_expected_qps = float(target_qps)
 
-    if samples_per_query_multistream:
-        settings.multi_stream_samples_per_query = samples_per_query_multistream
-
-    if max_latency:
-        settings.server_target_latency_ns = int(max_latency * NANO_SEC)
-        settings.multi_stream_expected_latency_ns = int(max_latency * NANO_SEC)
-
     # inference benchmark warmup
     if is_streaming:
         ds.init_sut()
@@ -549,7 +560,7 @@ def flush_queries() -> None:
     sut = lg.ConstructSUT(issue_queries, flush_queries)
     qsl = lg.ConstructQSL(
         count,
-        min(count, max_num_samples) if max_num_samples > 0 else count,
+        count,
         load_query_samples,
         ds.unload_query_samples,
     )
@@ -572,18 +583,28 @@ def flush_queries() -> None:
     if int(os.environ.get("WORLD_SIZE", 1)) > 1:
         model_family.predict(None)
 
-    if out_dir:
-        with open("results.json", "w") as f:
-            json.dump(final_results, f, sort_keys=True, indent=4)
-
 
 def main() -> None:
     set_verbose_level(1)
     args = get_args()
     logger.info(args)
-    gin_path = f"{os.path.dirname(__file__)}/gin/{SUPPORTED_CONFIGS[args.dataset]}"
-    gin.parse_config_file(gin_path)
-    run(dataset=args.dataset)
+    run(
+        dataset=args.dataset,
+        model_path=args.model_path,
+        scenario_name=args.scenario_name,
+        batchsize=args.batchsize,
+        output_trace=args.output_trace,
+        data_producer_threads=args.data_producer_threads,
+        compute_eval=args.compute_eval,
+        find_peak_performance=args.find_peak_performance,
+        dataset_path_prefix=args.dataset_path_prefix,
+        warmup_ratio=args.warmup_ratio,
+        target_qps=args.target_qps,
+        num_queries=args.num_queries,
+        numpy_rand_seed=args.numpy_rand_seed,
+        sparse_quant=args.sparse_quant,
+        dataset_percentage=args.dataset_percentage,
+    )
 
 
 if __name__ == "__main__":

recommendation/dlrm_v3/main.py

@@ -0,0 +1,6 @@
+torch==2.8.0
+fbgemm_gpu==1.3.0
+torchrec==1.3.0
+gin_config==0.5.0
+pandas==2.3.2
+tensorboard==2.20.0

recommendation/dlrm_v3/requirements.txt

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 WORLD_SIZE=8 \
+    python main.py --dataset sampled-streaming-100b 2>&1 | tee /home/$USER/dlrmv3-inference-benchmark.log

recommendation/dlrm_v3/run_benchmark.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+conda create --name dlrmv3 python=3.13
+conda activate dlrmv3
+pip install -r requirements.txt
+git_dir=$(git rev-parse --show-toplevel)
+pip install $git_dir/loadgen

recommendation/dlrm_v3/setup.sh

@@ -274,14 +274,6 @@ def reset(self, mode: str = "train"):
 
 # the datasets we support
 SUPPORTED_DATASETS = [
-    "debug",
-    "movielens-1m",
-    "movielens-20m",
-    "movielens-13b",
-    "movielens-18b",
-    "kuairand-1k",
-    "streaming-400m",
-    "streaming-200b",
     "streaming-100b",
     "sampled-streaming-100b",
 ]
@@ -290,38 +282,6 @@ def reset(self, mode: str = "train"):
 @gin.configurable
 def get_dataset(name: str, new_path_prefix: str = ""):
     assert name in SUPPORTED_DATASETS, f"dataset {name} not supported"
-    if name == "debug":
-        return DLRMv3RandomDataset, {}
-    if name == "streaming-400m":
-        return (
-            DLRMv3SyntheticStreamingDataset,
-            {
-                "ratings_file_prefix": os.path.join(
-                    new_path_prefix, "data/streaming-400m/"
-                ),
-                "train_ts": 8,
-                "total_ts": 10,
-                "num_files": 3,
-                "num_users": 150_000,
-                "num_items": 1_500_000,
-                "num_categories": 128,
-            },
-        )
-    if name == "streaming-200b":
-        return (
-            DLRMv3SyntheticStreamingDataset,
-            {
-                "ratings_file_prefix": os.path.join(
-                    new_path_prefix, "data/streaming-200b/"
-                ),
-                "train_ts": 90,
-                "total_ts": 100,
-                "num_files": 100,
-                "num_users": 10_000_000,
-                "num_items": 1_000_000_000,
-                "num_categories": 128,
-            },
-        )
     if name == "streaming-100b":
         return (
             DLRMv3SyntheticStreamingDataset,