Merge pull request #113743 from Yarwin/feature/improve-automated-checks-for-gdextension-compatibility

Improve automated checks for GDExtension compatibility

.github/actions/godot-compat-test/action.yml

@@ -0,0 +1,33 @@
+name: Godot hash compatibility test
+description: Check if methods with given hashes used by the older GDExtensions still can be loaded with given Godot version.
+
+inputs:
+  bin:
+    description: Path to the Godot binary.
+    required: true
+    type: string
+  reftags:
+    description: Reference tags of Godot versions to check (comma separated).
+    required: true
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: Extract GDExtension interface
+      shell: sh
+      run: |
+        ${{ inputs.bin }} --headless --dump-gdextension-interface
+        mkdir tests/compatibility_test/src/deps/
+        mv gdextension_interface.h tests/compatibility_test/src/deps/
+
+    - name: Build minimal GDExtension
+      shell: sh
+      run: scons --directory=./tests/compatibility_test
+
+    - name: Download reference GDExtension API JSON and try to load it
+      shell: sh
+      env:
+        GODOT4_BIN: ${{ inputs.bin }}
+        REFTAGS: ${{ inputs.reftags }}
+      run: ./tests/compatibility_test/run_compatibility_test.py

.github/workflows/linux_builds.yml

@@ -238,11 +238,18 @@ jobs:
           git diff --color --exit-code && ! git ls-files --others --exclude-standard | sed -e 's/^/New doc file missing in PR: /' | grep 'xml$'
 
       # Check API backwards compatibility
-      - name: Check for GDExtension compatibility
+      - name: Check for GDExtension compatibility – JSON check
         if: matrix.api-compat
         run: |
           ./misc/scripts/validate_extension_api.sh "${{ matrix.bin }}"
 
+      - name: Test GDExtension compatibility – load methods
+        uses: ./.github/actions/godot-compat-test
+        if: matrix.api-compat
+        with:
+          bin: ${{ matrix.bin }}
+          reftags: "4.5-stable,4.4-stable"
+
       # Download and run the test project
       - name: Test Godot project
         uses: ./.github/actions/godot-project-test

tests/compatibility_test/.gitignore

@@ -0,0 +1,2 @@
+godot/.godot
+src/deps/gdextension_interface.h

tests/compatibility_test/SConstruct

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# ruff: noqa: F821
+
+env = Environment()
+
+env.Append(CPPPATH=["src"])
+env.Append(CFLAGS=["-O0", "-g"])
+
+library = env.SharedLibrary(
+    target="#godot/compatibility_test",
+    source="#src/compat_checker.c",
+)
+
+env.Default(library)

tests/compatibility_test/godot/compatibility_test.gdextension

@@ -0,0 +1,6 @@
+[configuration]
+entry_symbol = "compatibility_test_init"
+compatibility_minimum = 4.1
+
+[libraries]
+linux.debug.x86_64 = "res://libcompatibility_test.so"

tests/compatibility_test/godot/compatibility_test.gdextension.uid

@@ -0,0 +1 @@
+uid://0h75rfqrneh3

tests/compatibility_test/godot/project.godot

@@ -0,0 +1,14 @@
+; Engine configuration file.
+; It's best edited using the editor UI and not directly,
+; since the parameters that go here are not all obvious.
+;
+; Format:
+;   [section] ; section goes between []
+;   param=value ; assign values to parameters
+
+config_version=5
+
+[application]
+
+config/name="CompatibilityTest"
+run/flush_stdout_on_print=true

tests/compatibility_test/run_compatibility_test.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import itertools
+import json
+import os
+import pathlib
+import subprocess
+import urllib.request
+from typing import Any
+
+PROJECT_PATH = pathlib.Path(__file__).parent.resolve().joinpath("godot")
+CLASS_METHODS_FILE = PROJECT_PATH.joinpath("class_methods.txt")
+BUILTIN_METHODS_FILE = PROJECT_PATH.joinpath("builtin_methods.txt")
+UTILITY_FUNCTIONS_FILE = PROJECT_PATH.joinpath("utility_functions.txt")
+
+
+def download_gdextension_api(reftag: str) -> dict[str, Any]:
+    with urllib.request.urlopen(
+        f"https://raw.githubusercontent.com/godotengine/godot-cpp/godot-{reftag}/gdextension/extension_api.json"
+    ) as f:
+        gdextension_api_json: dict[str, Any] = json.load(f)
+    return gdextension_api_json
+
+
+def remove_test_data_files():
+    for test_data in [CLASS_METHODS_FILE, BUILTIN_METHODS_FILE, UTILITY_FUNCTIONS_FILE]:
+        if os.path.isfile(test_data):
+            os.remove(test_data)
+
+
+def generate_test_data_files(reftag: str):
+    """
+    Parses methods specified in given Godot version into a form readable by the compatibility checker GDExtension.
+    """
+    gdextension_reference_json = download_gdextension_api(reftag)
+
+    with open(CLASS_METHODS_FILE, "w") as classes_file:
+        classes_file.writelines(
+            [
+                f"{klass['name']} {func['name']} {func['hash']}\n"
+                for (klass, func) in itertools.chain(
+                    (
+                        (klass, method)
+                        for klass in gdextension_reference_json["classes"]
+                        for method in klass.get("methods", [])
+                        if not method.get("is_virtual")
+                    ),
+                )
+            ]
+        )
+
+    variant_types: dict[str, int] | None = None
+    for global_enum in gdextension_reference_json["global_enums"]:
+        if global_enum.get("name") != "Variant.Type":
+            continue
+        variant_types = {
+            variant_type.get("name").removeprefix("TYPE_").lower().replace("_", ""): variant_type.get("value")
+            for variant_type in global_enum.get("values")
+        }
+
+    if not variant_types:
+        return
+
+    with open(BUILTIN_METHODS_FILE, "w") as f:
+        f.writelines(
+            [
+                f"{variant_types[klass['name'].lower()]} {func['name']} {func['hash']}\n"
+                for (klass, func) in itertools.chain(
+                    (
+                        (klass, method)
+                        for klass in gdextension_reference_json["builtin_classes"]
+                        for method in klass.get("methods", [])
+                    ),
+                )
+            ]
+        )
+
+    with open(UTILITY_FUNCTIONS_FILE, "w") as f:
+        f.writelines([f"{func['name']} {func['hash']}\n" for func in gdextension_reference_json["utility_functions"]])
+
+
+def has_compatibility_test_failed(errors: str) -> bool:
+    """
+    Checks if provided errors are related to the compatibility test.
+
+    Makes sure that test won't fail on unrelated account (for example editor misconfiguration).
+    """
+    compatibility_errors = [
+        "Error loading extension",
+        "Failed to load interface method",
+        'Parameter "mb" is null.',
+        'Parameter "bfi" is null.',
+        "Method bind not found:",
+        "Utility function not found:",
+        "has changed and no compatibility fallback has been provided",
+        "Failed to open file `builtin_methods.txt`",
+        "Failed to open file `class_methods.txt`",
+        "Failed to open file `utility_functions.txt`",
+        "Failed to open file `platform_methods.txt`",
+        "Outcome = FAILURE",
+    ]
+
+    return any(compatibility_error in errors for compatibility_error in compatibility_errors)
+
+
+def process_compatibility_test(proc: subprocess.Popen[bytes], timeout: int = 5) -> str | None:
+    """
+    Returns the stderr output as a string, if any.
+
+    Terminates test if nothing has been written to stdout/stderr for specified time.
+    """
+    errors = bytearray()
+
+    while True:
+        try:
+            _out, err = proc.communicate(timeout=timeout)
+            if err:
+                errors.extend(err)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            _out, err = proc.communicate()
+            if err:
+                errors.extend(err)
+            break
+
+    return errors.decode("utf-8") if errors else None
+
+
+def compatibility_check(godot4_bin: str) -> bool:
+    """
+    Checks if methods specified for previous Godot versions can be properly loaded with
+    the latest Godot4 binary.
+    """
+    # A bit crude albeit working solution – use stderr to check for compatibility-related errors.
+    proc = subprocess.Popen(
+        [godot4_bin, "--headless", "-e", "--path", PROJECT_PATH],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    if (errors := process_compatibility_test(proc)) and has_compatibility_test_failed(errors):
+        print(f"Compatibility test failed. Errors:\n {errors}")
+        return False
+    return True
+
+
+if __name__ == "__main__":
+    godot4_bin = os.environ["GODOT4_BIN"]
+    reftags = os.environ["REFTAGS"].split(",")
+    is_success = True
+    for reftag in reftags:
+        generate_test_data_files(reftag)
+        if not compatibility_check(godot4_bin):
+            print(f"Compatibility test against Godot{reftag} failed")
+            is_success = False
+        remove_test_data_files()
+
+    if not is_success:
+        exit(1)

tests/compatibility_test/src/compat_checker.c

@@ -0,0 +1,239 @@
+/**************************************************************************/
+/*  compat_checker.c                                                      */
+/**************************************************************************/
+/*                         This file is part of:                          */
+/*                             GODOT ENGINE                               */
+/*                        https://godotengine.org                         */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                  */
+/*                                                                        */
+/* Permission is hereby granted, free of charge, to any person obtaining  */
+/* a copy of this software and associated documentation files (the        */
+/* "Software"), to deal in the Software without restriction, including    */
+/* without limitation the rights to use, copy, modify, merge, publish,    */
+/* distribute, sublicense, and/or sell copies of the Software, and to     */
+/* permit persons to whom the Software is furnished to do so, subject to  */
+/* the following conditions:                                              */
+/*                                                                        */
+/* The above copyright notice and this permission notice shall be         */
+/* included in all copies or substantial portions of the Software.        */
+/*                                                                        */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,        */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF     */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY   */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,   */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE      */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
+/**************************************************************************/
+
+#include "compat_checker.h"
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+
+GDExtensionInterfaceClassdbGetMethodBind classdb_get_method_bind = NULL;
+GDExtensionInterfaceVariantGetPtrBuiltinMethod variant_get_ptr_builtin_method = NULL;
+GDExtensionInterfaceVariantGetPtrUtilityFunction variant_get_ptr_utility_function = NULL;
+
+GDExtensionPtrDestructor string_name_destructor = NULL;
+GDExtensionInterfaceStringNameNewWithLatin1Chars string_name_new_with_latin1_chars = NULL;
+
+typedef struct
+{
+	uint8_t data[8];
+} StringName;
+
+/**
+ * Platform APIs are being registered only after extensions, making them unavailable during initialization (on any level).
+ *
+ * Due to that we run the tests in Mainloop `startup_callback` (available since Godot 4.5), called after the initialization.
+ */
+void startup_func() {
+	bool success = (builtin_methods_compatibility_test() && class_methods_compatibility_test() && utility_functions_compatibility_test());
+	if (success) {
+		fprintf(stdout, "Outcome = SUCCESS\n");
+	} else {
+		fprintf(stderr, "Outcome = FAILURE\n");
+	}
+	fprintf(stdout, "COMPATIBILITY TEST FINISHED.\n");
+}
+
+GDExtensionMainLoopCallbacks callbacks = {
+	(GDExtensionMainLoopStartupCallback)startup_func,
+	NULL,
+	NULL
+};
+
+void initialize_compatibility_test(void *p_userdata, GDExtensionInitializationLevel p_level) {}
+
+void deinitialize_compatibility_test(void *p_userdata, GDExtensionInitializationLevel p_level) {}
+
+GDExtensionBool builtin_methods_compatibility_test() {
+	FILE *file = fopen("./builtin_methods.txt", "r");
+	if (file == NULL) {
+		fprintf(stderr, "Failed to open file `builtin_methods.txt` \n");
+		return false;
+	}
+
+	bool ret = true;
+	char line[512];
+
+	while (fgets(line, sizeof line, file) != NULL) {
+		int variant_type;
+		char method_name[128];
+		GDExtensionInt hash;
+		if (sscanf(line, "%d %s %ld", &variant_type, method_name, &hash) != 3) {
+			continue;
+		}
+
+		StringName method_stringname;
+		string_name_new_with_latin1_chars(&method_stringname, method_name, false);
+		GDExtensionPtrBuiltInMethod method_bind = variant_get_ptr_builtin_method(variant_type, &method_stringname, hash);
+
+		if (method_bind == NULL) {
+			fprintf(stderr, "Method bind not found: %d::%s (hash: %ld)\n", variant_type, method_name, hash);
+			ret = false;
+		}
+
+		string_name_destructor(&method_stringname);
+	}
+
+	fclose(file);
+	return ret;
+}
+
+GDExtensionBool utility_functions_compatibility_test() {
+	FILE *file = fopen("./utility_functions.txt", "r");
+	if (file == NULL) {
+		fprintf(stderr, "Failed to open file `utility_functions.txt` \n");
+		return false;
+	}
+
+	bool ret = true;
+	char line[256];
+
+	while (fgets(line, sizeof line, file) != NULL) {
+		char method_name[128];
+		GDExtensionInt hash;
+		if (sscanf(line, "%s %ld", method_name, &hash) != 2) {
+			continue;
+		}
+
+		StringName method_stringname;
+		string_name_new_with_latin1_chars(&method_stringname, method_name, false);
+		GDExtensionPtrUtilityFunction function_bind = variant_get_ptr_utility_function(&method_stringname, hash);
+
+		if (function_bind == NULL) {
+			fprintf(stderr, "Utility function not found: %s (hash: %ld)\n", method_name, hash);
+			ret = false;
+		}
+
+		string_name_destructor(&method_stringname);
+	}
+
+	fclose(file);
+	return ret;
+}
+
+GDExtensionBool class_methods_compatibility_test() {
+	FILE *file = fopen("./class_methods.txt", "r");
+	if (file == NULL) {
+		fprintf(stderr, "Failed to open file `class_methods.txt` \n");
+		return false;
+	}
+
+	char current_class_name[128] = "";
+	bool ret = true;
+	char line[512];
+	bool has_class_string = false;
+	StringName p_classname;
+
+	while (fgets(line, sizeof(line), file) != NULL) {
+		GDExtensionInt hash;
+		StringName p_methodname;
+		char class_name[128];
+		char method_name[128];
+
+		if (sscanf(line, "%s %s %ld", class_name, method_name, &hash) != 3) {
+			continue;
+		}
+
+		if (strcmp(current_class_name, class_name) != 0) {
+			if (has_class_string) {
+				string_name_destructor(&p_classname);
+			}
+			strcpy(current_class_name, class_name);
+
+			string_name_new_with_latin1_chars(&p_classname, current_class_name, false);
+
+			has_class_string = true;
+		}
+
+		string_name_new_with_latin1_chars(&p_methodname, method_name, false);
+		GDExtensionMethodBindPtr method_bind = classdb_get_method_bind(&p_classname, &p_methodname, hash);
+
+		if (method_bind == NULL) {
+			fprintf(stderr, "Method bind not found: %s.%s (hash: %ld)\n", class_name, method_name, hash);
+			ret = false;
+		}
+
+		string_name_destructor(&p_methodname);
+	}
+
+	if (has_class_string) {
+		string_name_destructor(&p_classname);
+	}
+
+	fclose(file);
+	return ret;
+}
+
+GDExtensionBool __attribute__((visibility("default"))) compatibility_test_init(GDExtensionInterfaceGetProcAddress p_get_proc_address, GDExtensionClassLibraryPtr p_library, GDExtensionInitialization *r_initialization) {
+	classdb_get_method_bind = (GDExtensionInterfaceClassdbGetMethodBind)p_get_proc_address("classdb_get_method_bind");
+	if (classdb_get_method_bind == NULL) {
+		fprintf(stderr, "Failed to load interface method `classdb_get_method_bind`\n");
+		return false;
+	}
+
+	variant_get_ptr_builtin_method = (GDExtensionInterfaceVariantGetPtrBuiltinMethod)p_get_proc_address("variant_get_ptr_builtin_method");
+	if (variant_get_ptr_builtin_method == NULL) {
+		fprintf(stderr, "Failed to load interface method `variant_get_ptr_builtin_method`\n");
+		return false;
+	}
+
+	variant_get_ptr_utility_function = (GDExtensionInterfaceVariantGetPtrUtilityFunction)p_get_proc_address("variant_get_ptr_utility_function");
+	if (variant_get_ptr_utility_function == NULL) {
+		fprintf(stderr, "Failed to load interface method `variant_get_ptr_utility_function`\n");
+		return false;
+	}
+
+	GDExtensionInterfaceVariantGetPtrDestructor variant_get_ptr_destructor = (GDExtensionInterfaceVariantGetPtrDestructor)p_get_proc_address("variant_get_ptr_destructor");
+	if (variant_get_ptr_destructor == NULL) {
+		fprintf(stderr, "Failed to load interface method `variant_get_ptr_destructor`\n");
+		return false;
+	}
+	string_name_destructor = variant_get_ptr_destructor(GDEXTENSION_VARIANT_TYPE_STRING_NAME);
+
+	string_name_new_with_latin1_chars = (GDExtensionInterfaceStringNameNewWithLatin1Chars)p_get_proc_address("string_name_new_with_latin1_chars");
+	if (classdb_get_method_bind == NULL) {
+		fprintf(stderr, "Failed to load interface method `string_name_new_with_latin1_chars`\n");
+		return false;
+	}
+
+	GDExtensionInterfaceRegisterMainLoopCallbacks register_main_loop_callbacks = (GDExtensionInterfaceRegisterMainLoopCallbacks)p_get_proc_address("register_main_loop_callbacks");
+	if (register_main_loop_callbacks == NULL) {
+		fprintf(stderr, "Failed to load interface method `register_main_loop_callbacks`\n");
+		return false;
+	}
+
+	register_main_loop_callbacks(p_library, &callbacks);
+
+	r_initialization->initialize = initialize_compatibility_test;
+	r_initialization->deinitialize = deinitialize_compatibility_test;
+	r_initialization->userdata = NULL;
+	r_initialization->minimum_initialization_level = GDEXTENSION_INITIALIZATION_EDITOR;
+
+	return true;
+}

tests/compatibility_test/src/compat_checker.h

@@ -0,0 +1,40 @@
+/**************************************************************************/
+/*  compat_checker.h                                                      */
+/**************************************************************************/
+/*                         This file is part of:                          */
+/*                             GODOT ENGINE                               */
+/*                        https://godotengine.org                         */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                  */
+/*                                                                        */
+/* Permission is hereby granted, free of charge, to any person obtaining  */
+/* a copy of this software and associated documentation files (the        */
+/* "Software"), to deal in the Software without restriction, including    */
+/* without limitation the rights to use, copy, modify, merge, publish,    */
+/* distribute, sublicense, and/or sell copies of the Software, and to     */
+/* permit persons to whom the Software is furnished to do so, subject to  */
+/* the following conditions:                                              */
+/*                                                                        */
+/* The above copyright notice and this permission notice shall be         */
+/* included in all copies or substantial portions of the Software.        */
+/*                                                                        */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,        */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF     */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY   */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,   */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE      */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
+/**************************************************************************/
+
+#pragma once
+
+#include "deps/gdextension_interface.h"
+
+void initialize_compatibility_test(void *p_userdata, GDExtensionInitializationLevel p_level);
+void deinitialize_compatibility_test(void *p_userdata, GDExtensionInitializationLevel p_level);
+GDExtensionBool class_methods_compatibility_test();
+GDExtensionBool builtin_methods_compatibility_test();
+GDExtensionBool utility_functions_compatibility_test();
+GDExtensionBool __attribute__((visibility("default"))) compatibility_test_init(GDExtensionInterfaceGetProcAddress p_get_proc_address, GDExtensionClassLibraryPtr p_library, GDExtensionInitialization *r_initialization);