Add new assets

.github/workflows/onpush.yaml

@@ -6,17 +6,17 @@ jobs:
   verify-files:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
     - name: Set up python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
       with:
         python-version: '3.11.9'
     - name: Install dependencies
       run: |
         python3 -m pip install --upgrade pip setuptools wheel
         pip3 install -r requirements.txt
     - name: Store python cache
-      uses: actions/cache@v4
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}

.gitignore

@@ -3,6 +3,7 @@ env/
 .vscode
 .idea
 .token_query_config.json
+.env
 .coingecko_cache
 global.db
 .DS_Store

docs/automation.md

@@ -1,5 +1,51 @@
 # Kraken asset automation workflow
 
+## Multi-exchange missing-assets compiler
+
+For warning files containing multiple exchanges, use the cache-first compiler:
+
+```bash
+.venv/bin/python tools/compile_missing_assets.py \
+  --missing missing.txt \
+  --version 41 \
+  --global-db /path/to/rotkehlchen/data/global.db \
+  --refresh
+```
+
+The compiler:
+
+- parses and deduplicates every supported exchange warning;
+- excludes symbols ending in `UP` or `DOWN`;
+- refreshes the large CoinGecko and CryptoCompare catalogues once, then reuses
+  them according to `--cache-max-age-hours`;
+- checks both the global database and pending update SQL before generating rows;
+- accepts unique exact CoinGecko symbol/name matches and leaves unsupported or
+  ambiguous matches unresolved;
+- supports evidence-backed decisions in
+  `updates/<version>/asset_compilation/reviewed_overrides.json`;
+- invokes the SQL generator per exchange and reconciles collection-main asset
+  mappings into SQL, JSON, and the root `mappings.csv`;
+- backfills missing EVM `started` timestamps using Blockscout PRO when
+  configured, then chain explorers and batched archive-RPC lookup, with all
+  successful results stored in a persistent deployment cache;
+- writes the full evidence manifest, resolved CSVs, ignored assets, and
+  unresolved assets under `updates/<version>/asset_compilation/`.
+
+Use `--resolve-only` to rebuild reports without changing SQL or mappings. Use
+`--offline` for a fully cached rerun. If CryptoCompare requires authentication,
+set `CRYPTOCOMPARE_API_KEY` and rerun with `--refresh`; until then, the manifest
+records the missing authentication and CryptoCompare IDs remain null.
+
+For authenticated Blockscout deployment lookups, create an ignored `.env` file
+at the repository root:
+
+```dotenv
+BLOCKSCOUT_API_KEY=your-key
+```
+
+`tools/backfill_started_dates.py` reads this file automatically and never
+writes the credential to generated output or the deployment cache.
+
 This document describes all scripts used in this asset-mapping workflow, the order to run them, and expected outputs.
 
 ## Goal

tests/test_generate_certain_sql_and_mappings.py

@@ -9,6 +9,7 @@
     find_existing_identifier_by_chain_address,
     find_existing_identifier_by_coingecko,
     load_existing_evm_address_map,
+    load_collection_main_by_symbol,
     load_next_collection_id,
     merge_location_json_additions,
     parse_certain_rows,
@@ -26,7 +27,7 @@ def test_parse_certain_rows_only_unique(tmp_path: Path) -> None:
         "CCC,2,ccc1|ccc2,C1|C2\n"
     )
 
-    assert parse_certain_rows(csv_path) == [("AAA", "aaa-token")]
+    assert parse_certain_rows(csv_path) == [("AAA", "aaa-token", None, None)]
 
 
 def test_extract_tokens_from_coin_dedupes_and_maps_supported_chains() -> None:
@@ -63,15 +64,17 @@ def test_choose_main_asset_prefers_ethereum() -> None:
 
 def test_token_sql_uses_started_timestamp() -> None:
     token = TokenRecord(
-        address="0xabc",
+        address="0x0000000000000000000000000000000000000abc",
         chain=Chain.ETHEREUM,
         decimals=18,
         name="Token",
         symbol="TOK",
         started=1700000000,
     )
-    sql = token_sql(token, coin_id="token-id", insert_or_ignore=True)
-    assert ", NULL, NULL, 1700000000, NULL);" in sql
+    sql = token_sql(token, coin_id="token-id", cryptocompare_id="TOK", insert_or_ignore=True)
+    assert "'TOK', NULL, 1700000000, NULL);" in sql
+    assert "'token-id', 'TOK'" in sql
+    assert "0x0000000000000000000000000000000000000aBc" in sql
 
 
 def test_find_existing_identifier_by_chain_address(tmp_path: Path) -> None:
@@ -140,6 +143,25 @@ def test_merge_location_json_additions_preserves_existing_and_dedupes() -> None:
     }
 
 
+def test_merge_location_json_additions_replaces_same_location_symbol() -> None:
+    merged = merge_location_json_additions(
+        [{"asset": "old", "location": "kraken", "location_symbol": "AAA"}],
+        [{"asset": "new", "location": "kraken", "location_symbol": "AAA"}],
+    )
+
+    assert merged == [{"asset": "new", "location": "kraken", "location_symbol": "AAA"}]
+
+
+def test_load_collection_main_by_symbol(tmp_path: Path) -> None:
+    path = tmp_path / "collections.sql"
+    path.write_text(
+        "INSERT INTO asset_collections(id, name, symbol, main_asset) "
+        "VALUES (1, 'Nesa', 'NES', 'eip155:56/erc20:0x123');\n"
+    )
+
+    assert load_collection_main_by_symbol(path) == {"NES": "eip155:56/erc20:0x123"}
+
+
 def test_load_next_collection_id_prefers_collections_sql(tmp_path: Path) -> None:
     collections_sql = tmp_path / "asset_collections_updates.sql"
     collections_sql.write_text(