Add support for Yul compilation and verification

packages/compilers-types/src/CompilationTypes.ts

@@ -1,6 +1,10 @@
-import { JsonFragment } from "ethers";
-import { SolidityOutputError, SoliditySettings } from "./SolidityTypes";
-import { VyperOutputError } from "./VyperTypes";
+import type { JsonFragment } from "ethers";
+import type {
+  SolidityJsonInput,
+  SolidityOutputError,
+  SoliditySettings,
+} from "./SolidityTypes";
+import type { VyperJsonInput, VyperOutputError } from "./VyperTypes";
 
 export interface LinkReferences {
   [filePath: string]: {
@@ -105,3 +109,5 @@ export interface Metadata {
   sources: MetadataSourceMap;
   version: number;
 }
+
+export type AnyJsonInput = SolidityJsonInput | VyperJsonInput;

packages/lib-sourcify/src/Compilation/CompilationTypes.ts

@@ -6,6 +6,8 @@ import type {
 } from '@ethereum-sourcify/compilers-types';
 import type { SourcifyLibErrorParameters } from '../SourcifyLibError';
 import { SourcifyLibError } from '../SourcifyLibError';
+import type { SolidityCompilation } from './SolidityCompilation';
+import type { VyperCompilation } from './VyperCompilation';
 
 export interface CompiledContractCborAuxdata {
   [key: string]: {
@@ -28,9 +30,10 @@ export interface CompilationTarget {
   path: string;
 }
 
-export type CompilationLanguage = 'Solidity' | 'Vyper';
+export type CompilationLanguage = 'Solidity' | 'Vyper' | 'Yul';
 
 export type CompilationErrorCode =
+  | 'invalid_language'
   | 'cannot_generate_cbor_auxdata_positions'
   | 'invalid_compiler_version'
   | 'unsupported_compiler_version'
@@ -66,3 +69,5 @@ export interface IVyperCompiler {
     vyperJsonInput: VyperJsonInput,
   ): Promise<VyperOutput>;
 }
+
+export type AnyCompilation = SolidityCompilation | VyperCompilation;

packages/lib-sourcify/src/Compilation/PreRunCompilation.ts

@@ -79,6 +79,7 @@ export class PreRunCompilation extends AbstractCompilation {
 
   get immutableReferences(): ImmutableReferences {
     switch (this.language) {
+      case 'Yul':
       case 'Solidity': {
         const compilationTarget = this
           .contractCompilerOutput as SolidityOutputContract;
@@ -97,6 +98,7 @@ export class PreRunCompilation extends AbstractCompilation {
 
   get runtimeLinkReferences(): LinkReferences {
     switch (this.language) {
+      case 'Yul':
       case 'Solidity': {
         const compilationTarget = this
           .contractCompilerOutput as SolidityOutputContract;
@@ -109,6 +111,7 @@ export class PreRunCompilation extends AbstractCompilation {
 
   get creationLinkReferences(): LinkReferences {
     switch (this.language) {
+      case 'Yul':
       case 'Solidity': {
         const compilationTarget = this
           .contractCompilerOutput as SolidityOutputContract;

packages/lib-sourcify/src/Compilation/VyperCompilation.ts

@@ -125,12 +125,18 @@ export class VyperCompilation extends AbstractCompilation {
       {},
     );
 
+    const {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      outputSelection: _outputSelection,
+      ...settingsWithoutOutputSelection
+    } = this.jsonInput.settings || {};
+
     this._metadata = {
       compiler: { version: this.compilerVersion },
       language: 'Vyper',
       output: outputMetadata,
       settings: {
-        ...this.jsonInput.settings,
+        ...settingsWithoutOutputSelection,
         compilationTarget: {
           [this.compilationTarget.path]: this.compilationTarget.name,
         },

packages/lib-sourcify/src/Compilation/YulCompilation.ts

@@ -0,0 +1,75 @@
+import type {
+  MetadataCompilerSettings,
+  SoliditySettings,
+} from '@ethereum-sourcify/compilers-types';
+import type { CompilationLanguage } from './CompilationTypes';
+import { SolidityCompilation } from './SolidityCompilation';
+import { id as keccak256str } from 'ethers';
+import { convertLibrariesToMetadataFormat } from '../utils/utils';
+
+/**
+ * Abstraction of a Yul compilation
+ */
+export class YulCompilation extends SolidityCompilation {
+  public language: CompilationLanguage = 'Yul';
+
+  public async compile(forceEmscripten = false) {
+    await this.compileAndReturnCompilationTarget(forceEmscripten);
+    this.generateMetadata();
+  }
+
+  /**
+   * Yul compiler does not produce a metadata but we generate it ourselves for backward
+   * compatibility reasons e.g. in the legacy Sourcify API that always assumes a metadata.json
+   */
+  generateMetadata() {
+    const contract = this.contractCompilerOutput;
+    const outputMetadata = {
+      abi: contract.abi,
+      devdoc: contract.devdoc,
+      userdoc: contract.userdoc,
+    };
+
+    const sourcesWithHashes = Object.entries(this.jsonInput.sources).reduce(
+      (acc, [path, source]) => ({
+        ...acc,
+        [path]: {
+          keccak256: keccak256str(source.content),
+        },
+      }),
+      {},
+    );
+
+    const soliditySettings = JSON.parse(
+      JSON.stringify(this.jsonInput.settings),
+    ) as SoliditySettings;
+
+    const {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      outputSelection: _outputSelection,
+      libraries,
+      ...settingsWithoutOutputSelection
+    } = soliditySettings;
+
+    const metadataSettings: Omit<
+      MetadataCompilerSettings,
+      'compilationTarget'
+    > = {
+      ...settingsWithoutOutputSelection,
+      libraries: convertLibrariesToMetadataFormat(libraries),
+    };
+    this._metadata = {
+      compiler: { version: this.compilerVersion },
+      language: 'Yul',
+      output: outputMetadata,
+      settings: {
+        ...metadataSettings,
+        compilationTarget: {
+          [this.compilationTarget.path]: this.compilationTarget.name,
+        },
+      },
+      sources: sourcesWithHashes,
+      version: 1,
+    };
+  }
+}

packages/lib-sourcify/src/Validation/SolidityMetadataContract.ts

@@ -3,7 +3,6 @@ import semver from 'semver';
 import { performFetch } from './fetchUtils';
 import { SolidityCompilation } from '../Compilation/SolidityCompilation';
 import type {
-  Libraries,
   SolidityJsonInput,
   Metadata,
   MetadataCompilerSettings,
@@ -33,7 +32,7 @@ import {
   getVariationsByContentHash,
 } from './variationsUtils';
 import { logDebug } from '../logger';
-import { splitFullyQualifiedName } from '../utils/utils';
+import { convertLibrariesToStdJsonFormat } from '../utils/utils';
 
 export class SolidityMetadataContract {
   metadata: Metadata;
@@ -317,32 +316,9 @@ export class SolidityMetadataContract {
 
     this.solcJsonInput.language = this.metadata.language;
 
-    // Convert the libraries from the metadata format to the compiler_settings format
-    // metadata format: "contracts/1_Storage.sol:Journal": "0x7d53f102f4d4aa014db4e10d6deec2009b3cda6b"
-    // settings format: "contracts/1_Storage.sol": { Journal: "0x7d53f102f4d4aa014db4e10d6deec2009b3cda6b" }
-    if (metadataLibraries) {
-      this.solcJsonInput.settings.libraries = Object.keys(
-        metadataLibraries,
-      ).reduce((libraries, libraryKey) => {
-        // Before Solidity v0.7.5: { "ERC20": "0x..."}
-        if (!libraryKey.includes(':')) {
-          if (!libraries['']) {
-            libraries[''] = {};
-          }
-          // try using the global method, available for pre 0.7.5 versions
-          libraries[''][libraryKey] = metadataLibraries[libraryKey];
-          return libraries;
-        }
-
-        // After Solidity v0.7.5: { "ERC20.sol:ERC20": "0x..."}
-        const { contractPath, contractName } =
-          splitFullyQualifiedName(libraryKey);
-        if (!libraries[contractPath]) {
-          libraries[contractPath] = {};
-        }
-        libraries[contractPath][contractName] = metadataLibraries[libraryKey];
-        return libraries;
-      }, {} as Libraries);
+    const libraries = convertLibrariesToStdJsonFormat(metadataLibraries);
+    if (libraries) {
+      this.solcJsonInput.settings.libraries = libraries;
     }
   }
 

packages/lib-sourcify/src/index.ts

@@ -10,6 +10,7 @@ export type ILibSourcifyLogger = ILogger;
 export * from './Compilation/AbstractCompilation';
 export * from './Compilation/SolidityCompilation';
 export * from './Compilation/VyperCompilation';
+export * from './Compilation/YulCompilation';
 export * from './Compilation/PreRunCompilation';
 export * from './Compilation/CompilationTypes';
 

packages/lib-sourcify/src/utils/utils.ts

@@ -1,3 +1,8 @@
+import type {
+  Libraries,
+  MetadataCompilerSettings,
+} from '@ethereum-sourcify/compilers-types';
+
 /**
  * Checks whether the provided object contains any keys or not.
  * @param obj The object whose emptiness is tested.
@@ -21,3 +26,62 @@ export function splitFullyQualifiedName(fullyQualifiedName: string): {
   const contractPath = splitIdentifier.slice(0, -1).join(':');
   return { contractPath, contractName };
 }
+
+/**
+ * Converts libraries from the solc JSON input format to the metadata format.
+ * jsonInput format: { "contracts/1_Storage.sol": { Journal: "0x..." } }
+ * metadata format: { "contracts/1_Storage.sol:Journal": "0x..." }
+ */
+export function convertLibrariesToMetadataFormat(
+  libraries?: Libraries,
+): MetadataCompilerSettings['libraries'] {
+  if (!libraries) {
+    return undefined;
+  }
+
+  const metadataLibraries: NonNullable<MetadataCompilerSettings['libraries']> =
+    {};
+
+  for (const [contractPath, libraryMap] of Object.entries(libraries)) {
+    for (const [libraryName, libraryAddress] of Object.entries(libraryMap)) {
+      const metadataKey =
+        contractPath === '' ? libraryName : `${contractPath}:${libraryName}`;
+      metadataLibraries[metadataKey] = libraryAddress;
+    }
+  }
+
+  return Object.keys(metadataLibraries).length ? metadataLibraries : undefined;
+}
+
+/**
+ * Converts libraries from the metadata format to the solc JSON input format.
+ * metadata format: { "contracts/1_Storage.sol:Journal": "0x..." }
+ * jsonInput format: { "contracts/1_Storage.sol": { Journal: "0x..." } }
+ */
+export function convertLibrariesToStdJsonFormat(
+  metadataLibraries?: MetadataCompilerSettings['libraries'],
+): Libraries | undefined {
+  if (!metadataLibraries) {
+    return undefined;
+  }
+
+  return Object.keys(metadataLibraries).reduce((libraries, libraryKey) => {
+    // Before Solidity v0.7.5: { "ERC20": "0x..."}
+    if (!libraryKey.includes(':')) {
+      if (!libraries['']) {
+        libraries[''] = {};
+      }
+      // try using the global method, available for pre 0.7.5 versions
+      libraries[''][libraryKey] = metadataLibraries[libraryKey];
+      return libraries;
+    }
+
+    // After Solidity v0.7.5: { "ERC20.sol:ERC20": "0x..."}
+    const { contractPath, contractName } = splitFullyQualifiedName(libraryKey);
+    if (!libraries[contractPath]) {
+      libraries[contractPath] = {};
+    }
+    libraries[contractPath][contractName] = metadataLibraries[libraryKey];
+    return libraries;
+  }, {} as Libraries);
+}

packages/lib-sourcify/test/Compilation/VyperCompilation.spec.ts

@@ -614,6 +614,9 @@ describe('VyperCompilation', () => {
 
     await compilation.compile();
 
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { outputSelection, ...settings } = compilation.jsonInput.settings;
+
     expect(compilation.metadata).to.deep.equal({
       compiler: { version: vyperVersion },
       language: 'Vyper',
@@ -636,7 +639,7 @@ describe('VyperCompilation', () => {
         userdoc: {},
       },
       settings: {
-        ...compilation.jsonInput.settings,
+        ...settings,
         compilationTarget: { [contractFileName]: contractName },
       },
       sources: {

packages/lib-sourcify/test/Compilation/YulCompilation.spec.ts

@@ -0,0 +1,74 @@
+import { describe, it } from 'mocha';
+import { expect, use } from 'chai';
+import fs from 'fs';
+import path from 'path';
+import { id as keccak256str } from 'ethers';
+import chaiAsPromised from 'chai-as-promised';
+import { YulCompilation } from '../../src/Compilation/YulCompilation';
+import { solc } from '../utils';
+import type { SolidityJsonInput } from '@ethereum-sourcify/compilers-types';
+
+use(chaiAsPromised);
+
+const compilerVersion = '0.8.26+commit.8a97fa7a';
+const contractName = 'cas-forwarder';
+const contractPath = 'cas-forwarder.yul';
+const fixturesBasePath = path.join(
+  __dirname,
+  '..',
+  'sources',
+  'Yul',
+  'cas-forwarder',
+);
+
+function loadJsonInput(): SolidityJsonInput {
+  const jsonInputPath = path.join(fixturesBasePath, 'jsonInput.json');
+  return JSON.parse(fs.readFileSync(jsonInputPath, 'utf8'));
+}
+
+describe('YulCompilation', () => {
+  it('should compile a Yul contract and generate metadata', async () => {
+    const jsonInput = loadJsonInput();
+
+    const compilation = new YulCompilation(solc, compilerVersion, jsonInput, {
+      name: contractName,
+      path: contractPath,
+    });
+
+    await compilation.compile(true);
+
+    expect(compilation.creationBytecode).to.equal(
+      '0x603780600a5f395ff3fe5f8080803560601c81813b9283923c818073ca11bde05977b3631167028862be2a173976ca115af13d90815f803e156034575ff35b5ffd',
+    );
+    expect(compilation.runtimeBytecode).to.equal(
+      '0x5f8080803560601c81813b9283923c818073ca11bde05977b3631167028862be2a173976ca115af13d90815f803e156034575ff35b5ffd',
+    );
+
+    const metadata = compilation.metadata;
+    expect(metadata.language).to.equal('Yul');
+    expect(metadata.compiler.version).to.equal(compilerVersion);
+    expect(metadata.settings.compilationTarget).to.deep.equal({
+      [contractPath]: contractName,
+    });
+
+    const expectedSourceHash = keccak256str(
+      jsonInput.sources[contractPath].content,
+    );
+    expect(metadata.sources[contractPath].keccak256).to.equal(
+      expectedSourceHash,
+    );
+  });
+
+  it('should throw when compilation target is invalid', async () => {
+    const jsonInput = loadJsonInput();
+
+    const compilation = new YulCompilation(solc, compilerVersion, jsonInput, {
+      name: 'non-existent',
+      path: 'wrong-path.yul',
+    });
+
+    await expect(compilation.compile(true)).to.be.rejectedWith(
+      'Contract not found in compiler output.',
+    );
+  });
+});