refactor: use CanonicalCodeMeta internal data model instead of CodeMeta

* build: need setuptools>=61.0 to build with only a pyproject.toml
* chore: add codemeta.json and CITATION.cff meta examples

Author: sgfost

CITATION.cff

@@ -0,0 +1,18 @@
+abstract: validate and convert between software metadata standards with pydantic
+authors:
+- affiliation: Arizona State University
+  email: [email protected]
+  family-names: Foster
+  given-names: Scott
+cff-version: 1.2.0
+keywords:
+- software metadata
+- codemeta
+- research software
+- FAIR
+license: GPL-3.0
+message: If you use this software, please cite it using the metadata from this file.
+repository-code: https://github.com/sgfost/codemeticulous/
+title: codemeticulous
+type: software
+version: 0.1.0

README.md

@@ -1,14 +1,11 @@
 ![](https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2Fsgfost%2Fcodemeticulous%2Fmain%2Fpyproject.toml) ![](https://img.shields.io/github/license/sgfost/codemeticulous)
 
-> [!NOTE]
+> [!WARNING]
 > `codemeticulous` is in an early state of development and things are subject to change. Refer to the [table](#feature-roadmap) below to see currently supported formats and conversions.
 
-`codemeticulous` is a python library and command line utility for validating and converting between different metadata standards for software. Validation is done by providing [pydantic](https://docs.pydantic.dev/latest/) models that mirror the standards' schema definitions.
+`codemeticulous` is a python library and command line utility for working with different metadata standards for software. Several [Pydantic](https://docs.pydantic.dev/latest/) models that mirror metadata schemas are provided which allows for simple validation, (de)serialization and type-safety for developers.
 
-Currently, CodeMeta is used as a central "hub" representation of software metadata as it is the most exhaustive, and provides [crosswalk definitions](https://codemeta.github.io/crosswalk/) between other formats. This is done in order to avoid the need for a bridge between every format, though custom conversion logic can be implemented where needed.
-
-> [!NOTE]
-> This is subject to change, however. There is an argument to be made for whether an even more robust internal data model would be beneficial. Namely, that going through CodeMeta/schema.org means some conversions will be lossy.
+For converting between different standards, an extension of [CodeMeta](https://codemeta.github.io/), called `CanonicalCodeMeta`, is used as a canonical data model or central "hub" representation, along with conversion logic back and forth between it and supported standards. This design allows for conversion between any two formats without needing to implement each bridge. CodeMeta was chosen as it is the most exhaustive and provides [crosswalk definitions](https://codemeta.github.io/crosswalk/) between other formats. Still, some data loss can occur, so some extension is needed to fill schema gaps and resolve abiguity. Note that `CanonicalCodeMeta` is not a proposed standard, but an internal data model used by this library.
 
 ## Feature Roadmap
 
@@ -96,15 +93,15 @@ $ codemeticulous validate --format cff CITATION.cff
 ### As a python library
 
 ```python
-from codemeticulous.codemeta.models import CodeMeta, Person
-from codemeticulous.cff.convert import codemeta_to_cff
+from codemeticulous.codemeta import CodeMeta, Person
+from codemeticulous import convert
 
 codemeta = CodeMeta(
   name="My Project",
   author=Person(givenName="Dale", familyName="Earnhardt"),
 )
 
-cff = codemeta_to_cff(codemeta)
+cff = convert("codemeta", "cff", codemeta)
 
 print(codemeta.json(indent=True))
 # {

codemeta.json

@@ -0,0 +1,34 @@
+{
+    "@context": "https://w3id.org/codemeta/3.0",
+    "type": "SoftwareSourceCode",
+    "author": [
+        {
+            "id": "_:author_1",
+            "type": "Person",
+            "affiliation": {
+                "type": "Organization",
+                "name": "Arizona State University"
+            },
+            "email": "[email protected]",
+            "familyName": "Foster",
+            "givenName": "Scott"
+        }
+    ],
+    "codeRepository": "https://github.com/sgfost/codemeticulous/",
+    "dateCreated": "2024-11-05",
+    "description": "validate and convert between software metadata standards with pydantic",
+    "keywords": [
+        "software metadata",
+        "codemeta",
+        "research software",
+        "FAIR"
+    ],
+    "license": "https://spdx.org/licenses/GPL-3.0",
+    "name": "codemeticulous",
+    "programmingLanguage": "Python 3",
+    "softwareRequirements": "Python 3.10",
+    "version": "0.1.0",
+    "continuousIntegration": "https://github.com/sgfost/codemeticulous/actions/",
+    "issueTracker": "https://github.com/sgfost/codemeticulous/issues"
+}
+

codemeticulous/__init__.py

@@ -0,0 +1,3 @@
+from .convert import convert, to_canonical, from_canonical
+
+__all__ = ["convert", "to_canonical", "from_canonical"]

codemeticulous/cff/__init__.py

@@ -1,3 +1,2 @@
-from .models import CitationFileFormat
-
-__all__ = ["CitationFileFormat"]
+from .models import *
+from .convert import *

codemeticulous/cff/convert.py

@@ -10,10 +10,8 @@
 from pydantic2_schemaorg.Person import Person as SchemaOrgPerson
 from pydantic2_schemaorg.Organization import Organization as SchemaOrgOrganization
 
-from codemeticulous.codemeta.extract import (
-    CodeMetaActorExtractor,
-    extract_doi_from_codemeta_identifier,
-)
+from codemeticulous.models import CanonicalCodeMeta
+from codemeticulous.extract import ActorExtractor, extract_doi_from_identifier
 from codemeticulous.codemeta.models import (
     CodeMeta,
     Actor as CodeMetaActor,
@@ -45,7 +43,7 @@ def codemeta_actors_to_cff(actors: CodeMetaActorListOrSingle) -> list[Person | E
     actors = ensure_list(actors)
     cff_actors = []
     for actor in actors:
-        extractor = CodeMetaActorExtractor(actor)
+        extractor = ActorExtractor(actor)
         if extractor.is_person:
             cff_actors.append(
                 Person(
@@ -213,12 +211,12 @@ def extract_main_url_from_codemeta(data: CodeMeta) -> str:
     )
 
 
-def codemeta_to_cff(data: CodeMeta) -> CitationFileFormat:
+def canonical_to_cff(data: CanonicalCodeMeta) -> CitationFileFormat:
     """Extract all possible Citation File Format fields from a CodeMeta object based
     on the CodeMeta crosswalk and return a CitationFileFormat object
     """
     licenses, license_urls = codemeta_license_to_cff(data.license)
-    primary_doi = extract_doi_from_codemeta_identifier(data.identifier)
+    primary_doi = extract_doi_from_identifier(data.identifier)
     return CitationFileFormat(
         cff_version="1.2.0",
         message="If you use this software, please cite it using the metadata from this file.",
@@ -250,5 +248,5 @@ def codemeta_to_cff(data: CodeMeta) -> CitationFileFormat:
     )
 
 
-def cff_to_codemeta(data: CitationFileFormat) -> CodeMeta:
+def cff_to_canonical(data: CitationFileFormat) -> CanonicalCodeMeta:
     raise NotImplementedError

codemeticulous/cli.py

@@ -1,34 +1,9 @@
 import os
 import click
 import json
-
 import yaml
 
-from codemeticulous.codemeta.models import CodeMeta
-from codemeticulous.datacite.models import DataciteV45
-from codemeticulous.cff.models import CitationFileFormat
-from codemeticulous.datacite.convert import codemeta_to_datacite, datacite_to_codemeta
-from codemeticulous.cff.convert import codemeta_to_cff, cff_to_codemeta
-
-
-models = {
-    "codemeta": {"model": CodeMeta, "format": "json"},
-    "datacite": {"model": DataciteV45, "format": "json"},
-    "cff": {"model": CitationFileFormat, "format": "yaml"},
-}
-
-converters = {
-    "codemeta": {
-        "datacite": codemeta_to_datacite,
-        "cff": codemeta_to_cff,
-    },
-    "datacite": {
-        "codemeta": datacite_to_codemeta,
-    },
-    "cff": {
-        "codemeta": cff_to_codemeta,
-    },
-}
+from codemeticulous.convert import STANDARDS, convert as _convert
 
 
 @click.group()
@@ -40,16 +15,16 @@ def cli():
 @click.option(
     "-f",
     "--from",
-    "from_format",
-    type=click.Choice(models.keys()),
+    "source_format",
+    type=click.Choice(STANDARDS.keys()),
     required=True,
     help="Source format",
 )
 @click.option(
     "-t",
     "--to",
-    "to_format",
-    type=click.Choice(models.keys()),
+    "target_format",
+    type=click.Choice(STANDARDS.keys()),
     required=True,
     help="Target format",
 )
@@ -62,27 +37,19 @@ def cli():
     help="Output file name (by default prints to stdout)",
 )
 @click.argument("input_file", type=click.Path(exists=True))
-def convert(from_format, to_format, input_file, output_file):
-    if to_format not in converters.get(from_format, {}):
-        click.echo(
-            f"Conversion from {from_format} to {to_format} is not supported", err=True
-        )
-        return
-
+def convert(source_format: str, target_format: str, input_file, output_file):
     try:
-        input_data = load_and_create_model(input_file, models[from_format]["model"])
-    except ValueError as e:
-        click.echo(str(e), err=True)
-        return
+        input_data = load_file_autodetect(input_file)
+    except Exception as e:
+        click.echo(f"Failed to load file: {input_file}. {str(e)}", err=True)
 
     try:
-        convert_func = converters[from_format][to_format]
-        converted_data = convert_func(input_data)
+        converted_data = _convert(source_format, target_format, input_data)
     except Exception as e:
         click.echo(f"Error during conversion: {str(e)}", err=True)
         return
 
-    output_format = models[to_format]["format"]
+    output_format = STANDARDS[target_format]["format"]
 
     try:
         output_data = dump_data(converted_data, output_format)
@@ -102,14 +69,14 @@ def convert(from_format, to_format, input_file, output_file):
     "-f",
     "--format",
     "format_name",
-    type=click.Choice(models.keys()),
+    type=click.Choice(STANDARDS.keys()),
     required=True,
     help="Format to validate",
 )
 @click.argument("input_file", type=click.Path(exists=True))
 def validate(format_name, input_file):
     try:
-        load_and_create_model(input_file, models[format_name]["model"])
+        load_and_create_model(input_file, STANDARDS[format_name]["model"])
         click.echo(f"{input_file} is a valid {format_name} file.")
     except ValueError as e:
         click.echo(str(e), err=True)
@@ -145,8 +112,6 @@ def load_file_autodetect(file_path):
             elif ext in [".yaml", ".yml", ".cff"]:
                 return yaml.safe_load(file)
             else:
-                raise ValueError(
-                    f"Unsupported file extension: {ext}."
-                )
+                raise ValueError(f"Unsupported file extension: {ext}.")
     except Exception as e:
         raise ValueError(f"Failed to load file: {file_path}. {str(e)}")

codemeticulous/codemeta/__init__.py

@@ -1,3 +1,2 @@
-from .models import CodeMeta
-
-__all__ = ["CodeMeta"]
+from .models import *
+from .convert import *

codemeticulous/codemeta/convert.py

@@ -0,0 +1,10 @@
+from codemeticulous.models import CanonicalCodeMeta
+from codemeticulous.codemeta.models import CodeMeta
+
+
+def canonical_to_codemeta(data: CanonicalCodeMeta) -> CodeMeta:
+    return CodeMeta(**data.dict())
+
+
+def codemeta_to_canonical(data: CodeMeta) -> CanonicalCodeMeta:
+    return CanonicalCodeMeta(**data.dict())

codemeticulous/convert.py

@@ -0,0 +1,55 @@
+from codemeticulous.codemeta.models import CodeMeta
+from codemeticulous.datacite.models import DataciteV45
+from codemeticulous.cff.models import CitationFileFormat
+from codemeticulous.codemeta.convert import canonical_to_codemeta, codemeta_to_canonical
+from codemeticulous.datacite.convert import canonical_to_datacite, datacite_to_canonical
+from codemeticulous.cff.convert import canonical_to_cff, cff_to_canonical
+
+
+STANDARDS = {
+    "codemeta": {
+        "model": CodeMeta,
+        "format": "json",
+        "to_canonical": codemeta_to_canonical,
+        "from_canonical": canonical_to_codemeta,
+    },
+    "datacite": {
+        "model": DataciteV45,
+        "format": "json",
+        "to_canonical": datacite_to_canonical,
+        "from_canonical": canonical_to_datacite,
+    },
+    "cff": {
+        "model": CitationFileFormat,
+        "format": "yaml",
+        "to_canonical": cff_to_canonical,
+        "from_canonical": canonical_to_cff,
+    },
+}
+
+
+def to_canonical(source_format: str, source_data):
+    source_model = STANDARDS[source_format]["model"]
+    if isinstance(source_data, dict):
+        source_instance = source_model(**source_data)
+    elif isinstance(source_data, source_model):
+        source_instance = source_data
+
+    source_to_canonical = STANDARDS[source_format]["to_canonical"]
+    canonical_instance = source_to_canonical(source_instance)
+
+    return canonical_instance
+
+
+def from_canonical(target_format: str, canonical_instance):
+    canonical_to_target = STANDARDS[target_format]["from_canonical"]
+    target_instance = canonical_to_target(canonical_instance)
+
+    return target_instance
+
+
+def convert(source_format: str, target_format: str, source_data):
+    # FIXME: add tons of error handling
+
+    canonical_instance = to_canonical(source_format, source_data)
+    return from_canonical(target_format, canonical_instance)