Add API for type libraries to declare named types without defining them

The C++, Python and Rust APIs can now declare a named type in a type
library without defining it, and list the types a library declares.

Previously Python's `NamedTypeReferenceType.target` and Rust's
`NamedTypeReference::target` looked up a reference only by its type id. A
reference to a declared type has no type under its id, so they now fall
back to the reference's name, matching core. Python gains
`BinaryView.get_type_by_ref` to do this.

Python's cycle detection when following references recorded the type
object rather than its id, and never recorded the starting id. A cycle
recursed until Python raised `RecursionError`. It now raises `TypeError`.

Author: bdash

binaryninjaapi.h

@@ -22165,6 +22165,12 @@ namespace BinaryNinja {
 		*/
 		std::vector<QualifiedNameAndType> GetNamedTypes();
 
+		/*! The type class a type library declares for \c name, or std::nullopt if it does not declare \c name. */
+		std::optional<BNNamedTypeReferenceClass> GetNamedTypeDeclaration(const QualifiedName& name);
+
+		/*! The named types a type library declares without defining, and the type class of each. */
+		std::vector<std::pair<QualifiedName, BNNamedTypeReferenceClass>> GetNamedTypeDeclarations();
+
 		/*! Sets the name of a type library instance that has not been finalized
 
 			\param name
@@ -22253,6 +22259,25 @@ namespace BinaryNinja {
 		*/
 		void AddNamedType(const QualifiedName& name, Ref<Type> type);
 
+		/*! Declares a named type without defining it, as C does with <tt>struct name;</tt>. Use it for a type
+			the library refers to only through pointers, such as an opaque struct whose layout comes from
+			the binary or another source. Importing the type into a view does not define it there. References
+			to it resolve to the view's type of that name and follow later renames. Defining a declared name
+			removes the declaration, and declaring a defined name does nothing.
+
+			\param name
+			\param typeClass The kind of type the name is: StructNamedTypeClass, ClassNamedTypeClass,
+				UnionNamedTypeClass or EnumNamedTypeClass. AddNamedTypeDeclaration logs an error and ignores other
+				classes.
+		*/
+		void AddNamedTypeDeclaration(const QualifiedName& name, BNNamedTypeReferenceClass typeClass);
+
+		/*! Removes a declaration made with AddNamedTypeDeclaration
+
+			\param name
+		*/
+		void RemoveNamedTypeDeclaration(const QualifiedName& name);
+
 		/*! Manually flag NamedTypeReferences to the given QualifiedName as originating from another source
 			TypeLibrary with the given dependency name.
 

binaryninjacore.h

@@ -2842,6 +2842,12 @@ extern "C"
 		BNType* type;
 	} BNQualifiedNameAndType;
 
+	typedef struct BNTypeLibraryNamedTypeDeclaration
+	{
+		BNQualifiedName name;
+		BNNamedTypeReferenceClass typeClass;
+	} BNTypeLibraryNamedTypeDeclaration;
+
 	typedef struct BNQualifiedNameTypeAndId
 	{
 		BNQualifiedName name;
@@ -7844,6 +7850,16 @@ extern "C"
 	BINARYNINJACOREAPI BNQualifiedNameAndType* BNGetTypeLibraryNamedObjects(BNTypeLibrary* lib, size_t* count);
 	BINARYNINJACOREAPI BNQualifiedNameAndType* BNGetTypeLibraryNamedTypes(BNTypeLibrary* lib, size_t* count);
 
+	BINARYNINJACOREAPI void BNAddTypeLibraryNamedTypeDeclaration(
+		BNTypeLibrary* lib, BNQualifiedName* name, BNNamedTypeReferenceClass typeClass);
+	BINARYNINJACOREAPI void BNRemoveTypeLibraryNamedTypeDeclaration(BNTypeLibrary* lib, BNQualifiedName* name);
+	BINARYNINJACOREAPI bool BNGetTypeLibraryNamedTypeDeclaration(
+		BNTypeLibrary* lib, BNQualifiedName* name, BNNamedTypeReferenceClass* typeClass);
+	BINARYNINJACOREAPI BNTypeLibraryNamedTypeDeclaration* BNGetTypeLibraryNamedTypeDeclarations(
+		BNTypeLibrary* lib, size_t* count);
+	BINARYNINJACOREAPI void BNFreeTypeLibraryNamedTypeDeclarationList(
+		BNTypeLibraryNamedTypeDeclaration* declarations, size_t count);
+
 	BINARYNINJACOREAPI bool BNWriteTypeLibraryToFile(BNTypeLibrary* lib, const char* path);
 
 	BINARYNINJACOREAPI void BNAddBinaryViewTypeLibrary(BNBinaryView* view, BNTypeLibrary* lib);

plugins/bntl_utils/src/process.rs

@@ -157,8 +157,11 @@ impl ProcessedData {
     ///
     /// This is useful if you intend to save the type libraries to disk in a finalized form.
     pub fn prune(self) -> Self {
-        let is_empty =
-            |tl: &TypeLibrary| tl.named_types().is_empty() && tl.named_objects().is_empty();
+        let is_empty = |tl: &TypeLibrary| {
+            tl.named_types().is_empty()
+                && tl.named_type_declarations().is_empty()
+                && tl.named_objects().is_empty()
+        };
         let pruned_type_libraries = self
             .type_libraries
             .into_iter()
@@ -205,6 +208,11 @@ impl ProcessedData {
                 for named_type in &tl.named_types() {
                     merged_type_library.add_named_type(named_type.name.clone(), &named_type.ty);
                 }
+                // A definition from any library wins over a declaration, whatever the merge order.
+                for declaration in &tl.named_type_declarations() {
+                    merged_type_library
+                        .add_named_type_declaration(declaration.name, declaration.type_class);
+                }
                 for named_object in &tl.named_objects() {
                     merged_type_library
                         .add_named_object(named_object.name.clone(), &named_object.ty);
@@ -298,7 +306,10 @@ impl ProcessedData {
                             // we can relocate them as well, assuming no other type library also uses it.
                             let is_associated = default_type_library
                                 .get_named_type(ntr_name.clone())
-                                .is_some();
+                                .is_some()
+                                || default_type_library
+                                    .get_named_type_declaration(ntr_name.clone())
+                                    .is_some();
                             if is_associated {
                                 recorded_references
                                     .entry(ntr_name)
@@ -327,10 +338,14 @@ impl ProcessedData {
             if type_libraries.len() == 1 {
                 // Only one type library uses this type, so we can safely relocate it.
                 let type_library = type_libraries.iter().next().unwrap();
-                let named_ty = default_type_library
-                    .get_named_type(qualified_name.clone())
-                    .unwrap();
-                type_library.add_named_type(qualified_name, &named_ty);
+                if let Some(named_ty) = default_type_library.get_named_type(qualified_name.clone())
+                {
+                    type_library.add_named_type(qualified_name, &named_ty);
+                } else if let Some(type_class) =
+                    default_type_library.get_named_type_declaration(qualified_name.clone())
+                {
+                    type_library.add_named_type_declaration(qualified_name, type_class);
+                }
             }
         }
     }
@@ -738,6 +753,14 @@ impl TypeLibProcessor {
             for t in tl.named_types().iter() {
                 type_name_to_library.insert(t.name.clone(), lib_name.clone());
             }
+            // Map declared names to the declaring library so the library being built records a type
+            // source instead of copying the view's definition. A library that defines the name takes
+            // precedence.
+            for declaration in &tl.named_type_declarations() {
+                type_name_to_library
+                    .entry(declaration.name)
+                    .or_insert_with(|| lib_name.clone());
+            }
         }
 
         let add_referenced_types = |type_library: &TypeLibrary, ty: &Type| {
@@ -1309,4 +1332,42 @@ mod tests {
         assert_eq!(merged_tl.platform_names().len(), 2);
         assert_eq!(merged_tl.named_types().len(), 2);
     }
+
+    #[test]
+    fn test_data_merging_keeps_declarations() {
+        use binaryninja::types::{NamedTypeDeclaration, NamedTypeReferenceClass};
+
+        let _session =
+            binaryninja::headless::Session::new().expect("Failed to create headless session");
+        let x86_platform = Platform::by_name("x86").expect("Failed to get x86 platform");
+        let tl1 = TypeLibrary::new(x86_platform.arch(), "foo");
+        tl1.set_dependency_name("foo");
+        tl1.add_platform(&x86_platform);
+        tl1.add_named_type_declaration(
+            "declared".into(),
+            NamedTypeReferenceClass::StructNamedTypeClass,
+        );
+        tl1.add_named_type_declaration(
+            "defined".into(),
+            NamedTypeReferenceClass::StructNamedTypeClass,
+        );
+        let tl1_data = ProcessedData::new(vec![tl1]);
+
+        let tl2 = TypeLibrary::new(x86_platform.arch(), "bar");
+        tl2.set_dependency_name("foo");
+        tl2.add_platform(&x86_platform);
+        tl2.add_named_type("defined".into(), &Type::named_int(4, false, "defined"));
+        let tl2_data = ProcessedData::new(vec![tl2]);
+
+        let merged_data = ProcessedData::merge(&[tl1_data, tl2_data]);
+        let merged_tl = &merged_data.type_libraries.iter().next().unwrap();
+        assert_eq!(
+            merged_tl.named_type_declarations().to_vec(),
+            vec![NamedTypeDeclaration {
+                name: "declared".into(),
+                type_class: NamedTypeReferenceClass::StructNamedTypeClass,
+            }]
+        );
+        assert!(merged_tl.get_named_type("defined".into()).is_some());
+    }
 }

plugins/bntl_utils/src/validate.rs

@@ -160,6 +160,9 @@ impl TypeLibValidater {
             for ty in &type_lib.named_types() {
                 self.valid_external_references.insert(ty.name);
             }
+            for declaration in &type_lib.named_type_declarations() {
+                self.valid_external_references.insert(declaration.name);
+            }
             for obj in &type_lib.named_objects() {
                 self.valid_external_references.insert(obj.name);
             }
@@ -247,11 +250,14 @@ impl TypeLibValidater {
 
             for ty in &types {
                 let qualified_name = QualifiedName::from(ty);
-                let is_named_ty = dep_type_lib
+                let is_resolved = dep_type_lib
                     .get_named_type(qualified_name.clone())
-                    .is_none();
-                let is_named_obj = dep_type_lib.get_named_object(qualified_name).is_none();
-                if !is_named_ty && !is_named_obj {
+                    .is_some()
+                    || dep_type_lib
+                        .get_named_type_declaration(qualified_name.clone())
+                        .is_some()
+                    || dep_type_lib.get_named_object(qualified_name).is_some();
+                if !is_resolved {
                     issues.push(ValidateIssue::UnresolvedSourceReference {
                         name: ty.to_string(),
                         source: src.to_string(),

python/binaryview.py

@@ -8986,6 +8986,20 @@ def get_type_by_id(self, id: str) -> Optional['_types.Type']:
 			return None
 		return _types.Type.create(obj, platform=self.platform)
 
+	def get_type_by_ref(self, ref: '_types.NamedTypeReferenceType') -> Optional['_types.Type']:
+		"""
+		``get_type_by_ref`` returns the type that ``ref`` refers to. It looks up the reference's id first,
+		then the reference's name.
+
+		:param NamedTypeReferenceType ref: Reference to resolve
+		:return: A :py:class:`Type` or None if no type is defined for the reference
+		:rtype: Type or None
+		"""
+		obj = core.BNGetAnalysisTypeByRef(self.handle, ref.ntr_handle)
+		if not obj:
+			return None
+		return _types.Type.create(obj, platform=self.platform)
+
 	def get_type_name_by_id(self, id: str) -> Optional['_types.QualifiedName']:
 		"""
 		``get_type_name_by_id`` returns the defined type name whose unique identifier corresponds with the provided ``id``

python/typelibrary.py

@@ -30,6 +30,7 @@
 from . import platform
 from . import architecture
 from . import typecontainer
+from .enums import NamedTypeReferenceClass
 
 
 class TypeLibrary:
@@ -428,6 +429,61 @@ def remove_named_type(self, name: Union[types.QualifiedName, str]) -> None:
 			name = types.QualifiedName(name)
 		core.BNRemoveTypeLibraryNamedType(self.handle, name._to_core_struct())
 
+	def add_named_type_declaration(
+		self, name: Union[types.QualifiedName, str], type_class: NamedTypeReferenceClass
+	) -> None:
+		"""
+		`add_named_type_declaration` declares a named type without defining it, as C does with ``struct name;``.
+		Use it for a type the library refers to only through pointers, such as an opaque struct whose layout comes
+		from the binary or another source. Importing the type into a view does not define it there. References to it
+		resolve to the view's type of that name and follow later renames. Defining a declared name removes the
+		declaration, and declaring a defined name does nothing.
+
+		:param QualifiedName name:
+		:param NamedTypeReferenceClass type_class: the kind of type the name is: ``StructNamedTypeClass``,
+			``ClassNamedTypeClass``, ``UnionNamedTypeClass`` or ``EnumNamedTypeClass``. `add_named_type_declaration`
+			logs an error and ignores other classes.
+		"""
+		if not isinstance(name, types.QualifiedName):
+			name = types.QualifiedName(name)
+		core.BNAddTypeLibraryNamedTypeDeclaration(self.handle, name._to_core_struct(), type_class)
+
+	def remove_named_type_declaration(self, name: Union[types.QualifiedName, str]) -> None:
+		"""
+		`remove_named_type_declaration` removes a declaration made with :py:meth:`add_named_type_declaration`.
+		"""
+		if not isinstance(name, types.QualifiedName):
+			name = types.QualifiedName(name)
+		core.BNRemoveTypeLibraryNamedTypeDeclaration(self.handle, name._to_core_struct())
+
+	def get_named_type_declaration(self, name: Union[types.QualifiedName, str]) -> Optional[NamedTypeReferenceClass]:
+		"""
+		`get_named_type_declaration` returns the type class the type library declares for ``name``, or None if it
+		does not declare ``name``.
+		"""
+		if not isinstance(name, types.QualifiedName):
+			name = types.QualifiedName(name)
+		type_class = core.NamedTypeReferenceClassEnum()
+		if not core.BNGetTypeLibraryNamedTypeDeclaration(self.handle, name._to_core_struct(), type_class):
+			return None
+		return NamedTypeReferenceClass(type_class.value)
+
+	@property
+	def named_type_declarations(self) -> Dict[types.QualifiedName, NamedTypeReferenceClass]:
+		"""
+		A dict of the named types the type library declares without defining, and the type class of each (read-only)
+		"""
+		count = ctypes.c_ulonglong(0)
+		declarations = core.BNGetTypeLibraryNamedTypeDeclarations(self.handle, count)
+		assert declarations is not None, "core.BNGetTypeLibraryNamedTypeDeclarations returned None"
+		try:
+			return {
+				types.QualifiedName._from_core_struct(declarations[i].name): NamedTypeReferenceClass(declarations[i].typeClass)
+				for i in range(count.value)
+			}
+		finally:
+			core.BNFreeTypeLibraryNamedTypeDeclarationList(declarations, count.value)
+
 	def add_type_source(self, name: Union[types.QualifiedName, str], source: str) -> None:
 		"""
 		Manually flag NamedTypeReferences to the given QualifiedName as originating from another source

python/types.py

@@ -3801,15 +3801,15 @@ def generate_auto_demangled_type_ref(type_class: NamedTypeReferenceClass, name:
 		return NamedTypeReferenceType.create(type_class, type_id, name)
 
 	def _target_helper(self, bv: 'binaryview.BinaryView', type_ids=None) -> Optional[Type]:
-		t = bv.get_type_by_id(self.type_id)
+		t = bv.get_type_by_ref(self)
 		if t is None:
 			return None
 		if type_ids is None:
-			type_ids = set()
+			type_ids = {self.type_id}
 		if isinstance(t, NamedTypeReferenceType):
 			if t.type_id in type_ids:
 				raise TypeError("Can't get target for recursively defined type")
-			type_ids.add(t)
+			type_ids.add(t.type_id)
 			return t._target_helper(bv, type_ids)
 		else:
 			return t

rust/src/types.rs

@@ -57,7 +57,7 @@ use std::{
 pub use archive::{TypeArchive, TypeArchiveId, TypeArchiveSnapshotId};
 pub use container::TypeContainer;
 pub use enumeration::{Enumeration, EnumerationBuilder, EnumerationMember};
-pub use library::TypeLibrary;
+pub use library::{NamedTypeDeclaration, TypeLibrary};
 pub use parser::{
     CoreTypeParser, ParsedType, TypeParser, TypeParserError, TypeParserErrorSeverity,
     TypeParserResult,
@@ -1627,7 +1627,7 @@ impl NamedTypeReference {
     }
 
     fn target_helper(&self, bv: &BinaryView, visited: &mut HashSet<String>) -> Option<Ref<Type>> {
-        let ty = bv.type_by_id(&self.id())?;
+        let ty = bv.type_by_ref(self)?;
         match ty.type_class() {
             TypeClass::NamedTypeReferenceClass => {
                 // Recurse into the NTR type until we get the target type.