From ba0c7db857cf36d03d7baf0a5a1b0ddc752746fa Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 25 Nov 2025 10:16:43 +0100 Subject: [PATCH 01/35] feat: minimal tombstone integration (disabled by default, options internal) --- gradle/libs.versions.toml | 4 + sentry-android-core/build.gradle.kts | 17 ++ .../core/AndroidOptionsInitializer.java | 5 + .../android/core/SentryAndroidOptions.java | 22 ++ .../android/core/TombstoneIntegration.java | 197 +++++++++++++++++ .../internal/tombstone/TombstoneParser.java | 194 +++++++++++++++++ .../core/internal/tombstone/tombstone.proto | 204 ++++++++++++++++++ .../internal/tombstone/TombstoneParserTest.kt | 108 ++++++++++ .../src/test/resources/tombstone.pb | Bin 0 -> 388771 bytes 9 files changed, 751 insertions(+) create mode 100644 sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java create mode 100644 sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java create mode 100644 sentry-android-core/src/main/proto/io/sentry/android/core/internal/tombstone/tombstone.proto create mode 100644 sentry-android-core/src/test/java/io/sentry/android/core/internal/tombstone/TombstoneParserTest.kt create mode 100644 sentry-android-core/src/test/resources/tombstone.pb diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 98b5cc37bfe..fc8219c4ec7 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -41,6 +41,7 @@ spotless = "7.0.4" gummyBears = "0.12.0" camerax = "1.3.0" openfeature = "1.18.2" +protobuf = "4.33.1" [plugins] kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } @@ -60,6 +61,7 @@ spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } detekt = { id = "io.gitlab.arturbosch.detekt", version = "1.23.8" } jacoco-android = { id = "com.mxalbert.gradle.jacoco-android", version = "0.2.0" } kover = { id = "org.jetbrains.kotlinx.kover", version = "0.7.3" } +protobuf = { id = "com.google.protobuf", version = "0.9.5" } vanniktech-maven-publish = { id = "com.vanniktech.maven.publish", version = "0.30.0" } springboot2 = { id = "org.springframework.boot", version.ref = "springboot2" } springboot3 = { id = "org.springframework.boot", version.ref = "springboot3" } @@ -138,6 +140,8 @@ otel-javaagent-extension-api = { module = "io.opentelemetry.javaagent:openteleme otel-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version.ref = "otelSemanticConventions" } otel-semconv-incubating = { module = "io.opentelemetry.semconv:opentelemetry-semconv-incubating", version.ref = "otelSemanticConventionsAlpha" } p6spy = { module = "p6spy:p6spy", version = "3.9.1" } +protobuf-javalite = { module = "com.google.protobuf:protobuf-javalite", version.ref = "protobuf"} +protoc = { module = "com.google.protobuf:protoc", version.ref = "protobuf" } quartz = { module = "org.quartz-scheduler:quartz", version = "2.3.0" } reactor-core = { module = "io.projectreactor:reactor-core", version = "3.5.3" } retrofit = { module = "com.squareup.retrofit2:retrofit", version.ref = "retrofit" } diff --git a/sentry-android-core/build.gradle.kts b/sentry-android-core/build.gradle.kts index 99d6b5115c8..3fce8891c7a 100644 --- a/sentry-android-core/build.gradle.kts +++ b/sentry-android-core/build.gradle.kts @@ -8,6 +8,7 @@ plugins { alias(libs.plugins.jacoco.android) alias(libs.plugins.errorprone) alias(libs.plugins.gradle.versions) + alias(libs.plugins.protobuf) } android { @@ -83,6 +84,7 @@ dependencies { implementation(libs.androidx.lifecycle.common.java8) implementation(libs.androidx.lifecycle.process) implementation(libs.androidx.core) + implementation(libs.protobuf.javalite) errorprone(libs.errorprone.core) errorprone(libs.nopen.checker) @@ -109,3 +111,18 @@ dependencies { testRuntimeOnly(libs.androidx.fragment.ktx) testRuntimeOnly(libs.timber) } + +protobuf { + protoc { + artifact = libs.protoc.get().toString() + } + generateProtoTasks { + all().forEach { task -> + task.builtins { + create("java") { + option("lite") + } + } + } + } +} diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java index 4e679a22e96..ef60b406ecd 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java @@ -5,6 +5,7 @@ import android.app.Application; import android.content.Context; import android.content.pm.PackageInfo; +import android.os.Build; import io.sentry.CompositePerformanceCollector; import io.sentry.DeduplicateMultithreadedEventProcessor; import io.sentry.DefaultCompositePerformanceCollector; @@ -372,6 +373,10 @@ static void installDefaultIntegrations( final Class sentryNdkClass = loadClass.loadClass(SENTRY_NDK_CLASS_NAME, options.getLogger()); options.addIntegration(new NdkIntegration(sentryNdkClass)); + if (buildInfoProvider.getSdkInfoVersion() >= Build.VERSION_CODES.R) { + options.addIntegration(new TombstoneIntegration(context)); + } + // this integration uses android.os.FileObserver, we can't move to sentry // before creating a pure java impl. options.addIntegration(EnvelopeFileObserverIntegration.getOutboxFileObserver()); diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java index 221495172eb..38fe7400c14 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java @@ -227,6 +227,8 @@ public interface BeforeCaptureCallback { private @Nullable SentryFrameMetricsCollector frameMetricsCollector; + private boolean tombstonesEnabled = false; + public SentryAndroidOptions() { setSentryClientName(BuildConfig.SENTRY_ANDROID_SDK_NAME + "/" + BuildConfig.VERSION_NAME); setSdkVersion(createSdkVersion()); @@ -300,6 +302,26 @@ public void setAnrReportInDebug(boolean anrReportInDebug) { this.anrReportInDebug = anrReportInDebug; } + /** + * Sets Tombstone reporting (ApplicationExitInfo.REASON_CRASH_NATIVE) to enabled or disabled. + * + * @param tombstonesEnabled true for enabled and false for disabled + */ + @ApiStatus.Internal + public void setTombstonesEnabled(boolean tombstonesEnabled) { + this.tombstonesEnabled = tombstonesEnabled; + } + + /** + * Checks if Tombstone reporting (ApplicationExitInfo.REASON_CRASH_NATIVE) is enabled or disabled Default is disabled + * + * @return true if enabled or false otherwise + */ + @ApiStatus.Internal + public boolean isTombstonesEnabled() { + return tombstonesEnabled; + } + public boolean isEnableActivityLifecycleBreadcrumbs() { return enableActivityLifecycleBreadcrumbs; } diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java new file mode 100644 index 00000000000..2ef46d64869 --- /dev/null +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -0,0 +1,197 @@ +package io.sentry.android.core; + +import static io.sentry.util.IntegrationUtils.addIntegrationToSdkVersion; + +import android.app.ActivityManager; +import android.app.ApplicationExitInfo; +import android.content.Context; +import android.os.Build; + +import androidx.annotation.RequiresApi; + +import io.sentry.DateUtils; +import io.sentry.IScopes; +import io.sentry.Integration; +import io.sentry.SentryEvent; +import io.sentry.SentryLevel; +import io.sentry.SentryOptions; +import io.sentry.android.core.internal.tombstone.TombstoneParser; +import io.sentry.cache.EnvelopeCache; +import io.sentry.cache.IEnvelopeCache; +import io.sentry.transport.CurrentDateProvider; +import io.sentry.transport.ICurrentDateProvider; +import io.sentry.util.Objects; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class TombstoneIntegration implements Integration, Closeable { + static final long NINETY_DAYS_THRESHOLD = TimeUnit.DAYS.toMillis(91); + + private final @NotNull Context context; + private final @NotNull ICurrentDateProvider dateProvider; + private @Nullable SentryAndroidOptions options; + + public TombstoneIntegration(final @NotNull Context context) { + // using CurrentDateProvider instead of AndroidCurrentDateProvider as AppExitInfo uses + // System.currentTimeMillis + this(context, CurrentDateProvider.getInstance()); + } + + TombstoneIntegration( + final @NotNull Context context, final @NotNull ICurrentDateProvider dateProvider) { + this.context = ContextUtils.getApplicationContext(context); + this.dateProvider = dateProvider; + } + + @Override + public void register(@NotNull IScopes scopes, @NotNull SentryOptions options) { + this.options = + Objects.requireNonNull( + (options instanceof SentryAndroidOptions) ? (SentryAndroidOptions) options : null, + "SentryAndroidOptions is required"); + + this.options + .getLogger() + .log(SentryLevel.DEBUG, "TombstoneIntegration enabled: %s", this.options.isTombstonesEnabled()); + + if (this.options.isTombstonesEnabled()) { + if (this.options.getCacheDirPath() == null) { + this.options + .getLogger() + .log(SentryLevel.INFO, "Cache dir is not set, unable to process Tombstones"); + return; + } + + try { + options + .getExecutorService() + .submit( + new TombstoneProcessor( + context, scopes, this.options, dateProvider)); + } catch (Throwable e) { + options.getLogger().log(SentryLevel.DEBUG, "Failed to start TombstoneProcessor.", e); + } + options.getLogger().log(SentryLevel.DEBUG, "TombstoneIntegration installed."); + addIntegrationToSdkVersion("Tombstone"); + } + } + + @Override + public void close() throws IOException { + if (options != null) { + options.getLogger().log(SentryLevel.DEBUG, "TombstoneIntegration removed."); + } + } + + public static class TombstoneProcessor implements Runnable { + + @NotNull + private final Context context; + @NotNull + private final IScopes scopes; + @NotNull + private final SentryAndroidOptions options; + private final long threshold; + + public TombstoneProcessor( + @NotNull Context context, + @NotNull IScopes scopes, + @NotNull SentryAndroidOptions options, + @NotNull ICurrentDateProvider dateProvider) { + this.context = context; + this.scopes = scopes; + this.options = options; + + this.threshold = dateProvider.getCurrentTimeMillis() - NINETY_DAYS_THRESHOLD; + } + + @Override + @RequiresApi(api = Build.VERSION_CODES.R) + public void run() { + final ActivityManager activityManager = + (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); + + final List applicationExitInfoList; + applicationExitInfoList = activityManager.getHistoricalProcessExitReasons(null, 0, 0); + + if (applicationExitInfoList.isEmpty()) { + options.getLogger().log(SentryLevel.DEBUG, "No records in historical exit reasons."); + return; + } + + final IEnvelopeCache cache = options.getEnvelopeDiskCache(); + if (cache instanceof EnvelopeCache) { + if (options.isEnableAutoSessionTracking() + && !((EnvelopeCache) cache).waitPreviousSessionFlush()) { + options + .getLogger() + .log( + SentryLevel.WARNING, + "Timed out waiting to flush previous session to its own file."); + + // if we timed out waiting here, we can already flush the latch, because the timeout is + // big + // enough to wait for it only once and we don't have to wait again in + // PreviousSessionFinalizer + ((EnvelopeCache) cache).flushPreviousSession(); + } + } + + // making a deep copy as we're modifying the list + final List exitInfos = new ArrayList<>(applicationExitInfoList); + + // search for the latest Tombstone to report it separately as we're gonna enrich it. The + // latest + // Tombstone will be first in the list, as it's filled last-to-first in order of appearance + ApplicationExitInfo latestTombstone = null; + for (ApplicationExitInfo applicationExitInfo : exitInfos) { + if (applicationExitInfo.getReason() == ApplicationExitInfo.REASON_CRASH_NATIVE) { + latestTombstone = applicationExitInfo; + // remove it, so it's not reported twice + // TODO: if we fail after this, we effectively lost the ApplicationExitInfo (maybe only remove after we reported it) + exitInfos.remove(applicationExitInfo); + break; + } + } + + if (latestTombstone == null) { + options + .getLogger() + .log( + SentryLevel.DEBUG, + "No Tombstones have been found in the historical exit reasons list."); + return; + } + + if (latestTombstone.getTimestamp() < threshold) { + options + .getLogger() + .log(SentryLevel.DEBUG, "Latest Tombstones happened too long ago, returning early."); + return; + } + + reportAsSentryEvent(latestTombstone); + } + + @RequiresApi(api = Build.VERSION_CODES.R) + private void reportAsSentryEvent(ApplicationExitInfo exitInfo) { + SentryEvent event; + try { + TombstoneParser parser = new TombstoneParser(exitInfo.getTraceInputStream()); + event = parser.parse(); + event.setTimestamp(DateUtils.getDateTime(exitInfo.getTimestamp())); + } catch (IOException e) { + throw new RuntimeException(e); + } + + scopes.captureEvent(event); + } + } +} diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java new file mode 100644 index 00000000000..ca1815c42b0 --- /dev/null +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -0,0 +1,194 @@ +package io.sentry.android.core.internal.tombstone; + +import androidx.annotation.NonNull; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import io.sentry.SentryEvent; +import io.sentry.SentryLevel; +import io.sentry.protocol.DebugImage; +import io.sentry.protocol.DebugMeta; +import io.sentry.protocol.Mechanism; +import io.sentry.protocol.Message; +import io.sentry.protocol.SentryException; +import io.sentry.protocol.SentryStackFrame; +import io.sentry.protocol.SentryStackTrace; +import io.sentry.protocol.SentryThread; + +public class TombstoneParser { + + private final InputStream tombstoneStream; + private final Map excTypeValueMap = new HashMap<>(); + + public TombstoneParser(InputStream tombstoneStream) { + this.tombstoneStream = tombstoneStream; + + // keep the current signal type -> value mapping for compatibility + excTypeValueMap.put("SIGILL", "IllegalInstruction"); + excTypeValueMap.put("SIGTRAP", "Trap"); + excTypeValueMap.put("SIGABRT", "Abort"); + excTypeValueMap.put("SIGBUS", "BusError"); + excTypeValueMap.put("SIGFPE", "FloatingPointException"); + excTypeValueMap.put("SIGSEGV", "Segfault"); + } + + public SentryEvent parse() throws IOException { + TombstoneProtos.Tombstone tombstone = TombstoneProtos.Tombstone.parseFrom(tombstoneStream); + + SentryEvent event = new SentryEvent(); + event.setLevel(SentryLevel.FATAL); + + // we must use the "native" platform because otherwise the stack-trace would not be correctly parsed + event.setPlatform("native"); + + event.setMessage(constructMessage(tombstone)); + event.setDebugMeta(createDebugMeta(tombstone)); + event.setExceptions(createException(tombstone)); + assert event.getExceptions() != null; + event.setThreads(createThreads(tombstone, event.getExceptions().get(0))); + + return event; + } + + @NonNull + private List createThreads(TombstoneProtos.Tombstone tombstone, SentryException exc) { + List threads = new ArrayList<>(); + for (Map.Entry threadEntry : + tombstone.getThreadsMap().entrySet()) { + + SentryThread thread = new SentryThread(); + thread.setId(Long.valueOf(threadEntry.getKey())); + thread.setName(threadEntry.getValue().getName()); + + SentryStackTrace stacktrace = createStackTrace(threadEntry); + thread.setStacktrace(stacktrace); + if (tombstone.getTid() == threadEntry.getValue().getId()) { + thread.setCrashed(true); + // even though we refer to the thread_id from the exception, the backend currently requires a stack-trace in exception + exc.setStacktrace(stacktrace); + } + threads.add(thread); + } + + return threads; + } + + @NonNull + private static SentryStackTrace createStackTrace(Map.Entry threadEntry) { + List frames = new ArrayList<>(); + + for (TombstoneProtos.BacktraceFrame frame : + threadEntry.getValue().getCurrentBacktraceList()) { + SentryStackFrame stackFrame = new SentryStackFrame(); + stackFrame.setPackage(frame.getFileName()); + stackFrame.setFunction(frame.getFunctionName()); + stackFrame.setInstructionAddr(String.format("0x%x", frame.getPc())); + frames.add(0, stackFrame); + } + + SentryStackTrace stacktrace = new SentryStackTrace(); + stacktrace.setFrames(frames); + + Map unknown = new HashMap<>(); + // `libunwindstack` used for tombstone generation already applies instruction address adjustment: + // https://android.googlesource.com/platform/system/unwinding/+/refs/heads/main/libunwindstack/Regs.cpp#175 + // prevent "processing" from doing it again. + unknown.put("instruction_addr_adjustment", "none"); + stacktrace.setUnknown(unknown); + + Map registers = new HashMap<>(); + for (TombstoneProtos.Register register : threadEntry.getValue().getRegistersList()) { + registers.put(register.getName(), String.format("0x%x", register.getU64())); + } + stacktrace.setRegisters(registers); + + return stacktrace; + } + + @NonNull + private List createException(TombstoneProtos.Tombstone tombstone) { + TombstoneProtos.Signal signalInfo = tombstone.getSignalInfo(); + + SentryException exception = new SentryException(); + exception.setType(signalInfo.getName()); + exception.setValue(excTypeValueMap.get(signalInfo.getName())); + exception.setMechanism(createMechanismFromSignalInfo(signalInfo)); + exception.setThreadId((long) tombstone.getTid()); + + List exceptions = new ArrayList<>(); + exceptions.add(exception); + + return exceptions; + } + + @NonNull + private static Mechanism createMechanismFromSignalInfo(TombstoneProtos.Signal signalInfo) { + Map meta = new HashMap<>(); + meta.put("number", signalInfo.getNumber()); + meta.put("name", signalInfo.getName()); + meta.put("code", signalInfo.getCode()); + meta.put("code_name", signalInfo.getCodeName()); + + Mechanism mechanism = new Mechanism(); + // this follows the current processing triggers strictly, changing any of these alters grouping and name (long-term we might want to + // have a tombstone mechanism) + mechanism.setType("signalhandler"); + mechanism.setHandled(false); + mechanism.setSynthetic(true); + mechanism.setMeta(meta); + + return mechanism; + } + + @NonNull + private Message constructMessage(TombstoneProtos.Tombstone tombstone) { + Message message = new Message(); + TombstoneProtos.Signal signalInfo = tombstone.getSignalInfo(); + + // reproduce the message `debuggerd` would use to dump the stack trace in logcat + message.setFormatted( + String.format(Locale.getDefault(), + "Fatal signal %s (%d), %s (%d), pid = %d (%s)", + signalInfo.getName(), + signalInfo.getNumber(), + signalInfo.getCodeName(), + signalInfo.getCode(), + tombstone.getPid(), + String.join(" ", tombstone.getCommandLineList()))); + + return message; + } + + private DebugMeta createDebugMeta(TombstoneProtos.Tombstone tombstone) { + List images = new ArrayList<>(); + + for (TombstoneProtos.MemoryMapping module : tombstone.getMemoryMappingsList()) { + // exclude anonymous and non-executable maps + if (module.getBuildId().isEmpty() + || module.getMappingName().isEmpty() + || !module.getExecute()) { + continue; + } + DebugImage image = new DebugImage(); + image.setCodeId(module.getBuildId()); + image.setCodeFile(module.getMappingName()); + image.setDebugId(module.getBuildId()); + image.setImageAddr(String.format("0x%x", module.getBeginAddress())); + image.setImageSize(module.getEndAddress() - module.getBeginAddress()); + image.setType("elf"); + + images.add(image); + } + + DebugMeta debugMeta = new DebugMeta(); + debugMeta.setImages(images); + + return debugMeta; + } +} diff --git a/sentry-android-core/src/main/proto/io/sentry/android/core/internal/tombstone/tombstone.proto b/sentry-android-core/src/main/proto/io/sentry/android/core/internal/tombstone/tombstone.proto new file mode 100644 index 00000000000..c75eae32688 --- /dev/null +++ b/sentry-android-core/src/main/proto/io/sentry/android/core/internal/tombstone/tombstone.proto @@ -0,0 +1,204 @@ +// Added and adapted from: https://android.googlesource.com/platform/system/core/+/refs/heads/main/debuggerd/proto/tombstone.proto +// Sentry changes: +// * change the java_package +// +// Protobuf definition for Android tombstones. +// +// An app can get hold of these for any `REASON_CRASH_NATIVE` instance of +// `android.app.ApplicationExitInfo`. +// +// https://developer.android.com/reference/android/app/ApplicationExitInfo#getTraceInputStream() +// +syntax = "proto3"; +option java_package = "io.sentry.android.core.internal.tombstone"; +option java_outer_classname = "TombstoneProtos"; +// NOTE TO OEMS: +// If you add custom fields to this proto, do not use numbers in the reserved range. +// NOTE TO CONSUMERS: +// With proto3 -- unlike proto2 -- HasValue is unreliable for any field +// where the default value for that type is also a valid value for the field. +// This means, for example, that a boolean that is false or an integer that +// is zero will appear to be missing --- but because they're not actually +// marked as `optional` in this schema, consumers should just use values +// without first checking whether or not they're "present". +// https://protobuf.dev/programming-guides/proto3/#default +message CrashDetail { + bytes name = 1; + bytes data = 2; + reserved 3 to 999; +} +message StackHistoryBufferEntry { + BacktraceFrame addr = 1; + uint64 fp = 2; + uint64 tag = 3; + reserved 4 to 999; +} +message StackHistoryBuffer { + uint64 tid = 1; + repeated StackHistoryBufferEntry entries = 2; + reserved 3 to 999; +} +message Tombstone { + Architecture arch = 1; + Architecture guest_arch = 24; + string build_fingerprint = 2; + string revision = 3; + string timestamp = 4; + uint32 pid = 5; + uint32 tid = 6; + uint32 uid = 7; + string selinux_label = 8; + repeated string command_line = 9; + // Process uptime in seconds. + uint32 process_uptime = 20; + Signal signal_info = 10; + string abort_message = 14; + repeated CrashDetail crash_details = 21; + repeated Cause causes = 15; + map threads = 16; + map guest_threads = 25; + repeated MemoryMapping memory_mappings = 17; + repeated LogBuffer log_buffers = 18; + repeated FD open_fds = 19; + uint32 page_size = 22; + bool has_been_16kb_mode = 23; + StackHistoryBuffer stack_history_buffer = 26; + reserved 27 to 999; +} +enum Architecture { + ARM32 = 0; + ARM64 = 1; + X86 = 2; + X86_64 = 3; + RISCV64 = 4; + NONE = 5; + reserved 6 to 999; +} +message Signal { + int32 number = 1; + string name = 2; + int32 code = 3; + string code_name = 4; + bool has_sender = 5; + int32 sender_uid = 6; + int32 sender_pid = 7; + bool has_fault_address = 8; + uint64 fault_address = 9; + // Note, may or may not contain the dump of the actual memory contents. Currently, on arm64, we + // only include metadata, and not the contents. + MemoryDump fault_adjacent_metadata = 10; + reserved 11 to 999; +} +message HeapObject { + uint64 address = 1; + uint64 size = 2; + uint64 allocation_tid = 3; + repeated BacktraceFrame allocation_backtrace = 4; + uint64 deallocation_tid = 5; + repeated BacktraceFrame deallocation_backtrace = 6; +} +message MemoryError { + enum Tool { + GWP_ASAN = 0; + SCUDO = 1; + reserved 2 to 999; + } + Tool tool = 1; + enum Type { + UNKNOWN = 0; + USE_AFTER_FREE = 1; + DOUBLE_FREE = 2; + INVALID_FREE = 3; + BUFFER_OVERFLOW = 4; + BUFFER_UNDERFLOW = 5; + reserved 6 to 999; + } + Type type = 2; + oneof location { + HeapObject heap = 3; + } + reserved 4 to 999; +} +message Cause { + string human_readable = 1; + oneof details { + MemoryError memory_error = 2; + } + reserved 3 to 999; +} +message Register { + string name = 1; + uint64 u64 = 2; + reserved 3 to 999; +} +message Thread { + int32 id = 1; + string name = 2; + repeated Register registers = 3; + repeated string backtrace_note = 7; + repeated string unreadable_elf_files = 9; + repeated BacktraceFrame current_backtrace = 4; + repeated MemoryDump memory_dump = 5; + int64 tagged_addr_ctrl = 6; + int64 pac_enabled_keys = 8; + reserved 10 to 999; +} +message BacktraceFrame { + uint64 rel_pc = 1; + uint64 pc = 2; + uint64 sp = 3; + string function_name = 4; + uint64 function_offset = 5; + string file_name = 6; + uint64 file_map_offset = 7; + string build_id = 8; + reserved 9 to 999; +} +message ArmMTEMetadata { + // One memory tag per granule (e.g. every 16 bytes) of regular memory. + bytes memory_tags = 1; + reserved 2 to 999; +} +message MemoryDump { + string register_name = 1; + string mapping_name = 2; + uint64 begin_address = 3; + bytes memory = 4; + oneof metadata { + ArmMTEMetadata arm_mte_metadata = 6; + } + reserved 5, 7 to 999; +} +message MemoryMapping { + uint64 begin_address = 1; + uint64 end_address = 2; + uint64 offset = 3; + bool read = 4; + bool write = 5; + bool execute = 6; + string mapping_name = 7; + string build_id = 8; + uint64 load_bias = 9; + reserved 10 to 999; +} +message FD { + int32 fd = 1; + string path = 2; + string owner = 3; + uint64 tag = 4; + reserved 5 to 999; +} +message LogBuffer { + string name = 1; + repeated LogMessage logs = 2; + reserved 3 to 999; +} +message LogMessage { + string timestamp = 1; + uint32 pid = 2; + uint32 tid = 3; + uint32 priority = 4; + string tag = 5; + string message = 6; + reserved 7 to 999; +} diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/internal/tombstone/TombstoneParserTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/internal/tombstone/TombstoneParserTest.kt new file mode 100644 index 00000000000..3f321dc4d80 --- /dev/null +++ b/sentry-android-core/src/test/java/io/sentry/android/core/internal/tombstone/TombstoneParserTest.kt @@ -0,0 +1,108 @@ +package io.sentry.android.core.internal.tombstone + +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class TombstoneParserTest { + val expectedRegisters = setOf( + "x8", + "x9", + "esr", + "lr", + "pst", + "x10", + "x12", + "x11", + "x14", + "x13", + "x16", + "x15", + "sp", + "x18", + "x17", + "x19", + "pc", + "x21", + "x20", + "x0", + "x23", + "x1", + "x22", + "x2", + "x25", + "x3", + "x24", + "x4", + "x27", + "x5", + "x26", + "x6", + "x29", + "x7", + "x28" + ) + + @Test + fun `parses a snapshot tombstone into Event`() { + val tombstone = File("src/test/resources/tombstone.pb") + val parser = TombstoneParser(tombstone.inputStream()) + val event = parser.parse() + + // top-level data + assertNotNull(event.eventId) + assertEquals("Fatal signal SIGSEGV (11), SEGV_MAPERR (1), pid = 21891 (io.sentry.samples.android)", event.message!!.formatted) + assertEquals("native", event.platform) + assertEquals("FATAL", event.level!!.name) + + // exception + // we only track one native exception (no nesting, one crashed thread) + assertEquals(1, event.exceptions!!.size) + val exception = event.exceptions!![0] + assertEquals("SIGSEGV", exception.type) + assertEquals("Segfault", exception.value) + val crashedThreadId = exception.threadId + assertNotNull(crashedThreadId) + + val mechanism = exception.mechanism + assertEquals("signalhandler", mechanism!!.type) + assertEquals(false, mechanism.isHandled) + assertEquals(true, mechanism.synthetic) + assertEquals("SIGSEGV", mechanism.meta!!["name"]) + assertEquals(11, mechanism.meta!!["number"]) + assertEquals("SEGV_MAPERR", mechanism.meta!!["code_name"]) + assertEquals(1, mechanism.meta!!["code"]) + + // threads + assertEquals(62, event.threads!!.size) + for (thread in event.threads!!) { + assertNotNull(thread.id) + if (thread.id == crashedThreadId) { + assert(thread.isCrashed == true) + } + assert(thread.stacktrace!!.frames!!.isNotEmpty()) + + for (frame in thread.stacktrace!!.frames!!) { + assertNotNull(frame.function) + assertNotNull(frame.`package`) + assertNotNull(frame.instructionAddr) + } + + assert(thread.stacktrace!!.registers!!.keys.containsAll(expectedRegisters)) + } + + // debug-meta + assertEquals(357, event.debugMeta!!.images!!.size) + for (image in event.debugMeta!!.images!!) { + assertEquals("elf", image.type) + assertNotNull(image.debugId) + assertNotNull(image.codeId) + assertEquals(image.codeId, image.debugId) + assertNotNull(image.codeFile) + val imageAddress = image.imageAddr!!.removePrefix("0x").toLong(16) + assert(imageAddress > 0) + assert(image.imageSize!! > 0) + } + } +} diff --git a/sentry-android-core/src/test/resources/tombstone.pb b/sentry-android-core/src/test/resources/tombstone.pb new file mode 100644 index 0000000000000000000000000000000000000000..051356a8fefa9baee4645ea56618afd7f5a273ca GIT binary patch literal 388771 zcmeEv3wTpiwm)fk9MqAm-wS#$$ z(V#JD4dz;(=pF18U82<|4jM1k`TF;|ZWLdxRw@Oxpc};Vg9L3UuQv;NGjA+cs|{Ls zOk?_$npdk;QvEOWc${v(=qf!#KLUB)DkvozMoH_-aUq4Jw?}*vJmn5$>$w z8lRs8O8WzYqR*@Ja65MXQo35go>2Dj)%k2zmrM0Cp?t8lMw~R*=Bbw7m2xk2#u$n}hLu=6Kc~NWR^&+plPr=Pu zGetq>AgeM-v(@W2o8hgoJN?k&O;)edTHzAS<{N>OeXP~#SK%EuR(VCM{gQ#DE|0r% zU@5$h15}Mg!hf`76Kg+e{i)}?Xa82IsJodR$G-=sxNoN}^5J z){kggH`h9C<1J421kZTU;`7&5sP^>iye@pH;HJHCZ!wwn)c(7HaCj*azC%HWkBuIx zqZ%*p3L`z-GYsC@OC461PrQu1EknKj>qLK*hrTc=!!4hUsj}KV6R(895PvD%Fv9Hv zO$v(3<1V#%-~{fib^Dx^ZqZ)qtMYjLX+PgJH(5dDUot?o?3~W)_j3xa_3?&c(t7(x zwBNDEu;0I%0E#+Gx38Ea+G_pcn2Aomt!f0Q^=jbcHJ?~I>8E>W3cq-IICB=u!viBS42UUeTBV~2t^e5YNZXc)Zq-B zh4&5f*u@ckv3j`GZyjp0i9Vml%U(749nA7tviD4W^{4<0C{>kPEVXXu&E-5VXf--5 zZ&dTVLC2eTy}=|Hd6Pk>GwJjO4J-q6g2q_xa!sf%*BUhjt=hyJ4SKCXqvv&c-2m0b zPSl`(%XDO;zrL1x`bddd%4$XuM|~>I`RGc2e)G5tML9U zyO+Ly!Do>AML=EBReless{{j<3dX_jI`36{Z3ao z=oDM67nV-`a;Ey}i^$#mtmk|V zaB(}T$c~nK=@bx%Fq$X~E~~pz5;oNL!~SS>d)#Kb)iuF6eo*_$-{$d(gFNHPLHiGa zMU&g-@OWW8`lrmPKq7c%Do9&Qr%wgJRrsTH)l{$sspoSlNY|P1wa7pobmj{RZbcNP z;1%K9=_D9ghlKwJS06*=6|vbo0+;O>OO(V~7iP)Sr7xUHD`aWOJFwJNWtA2#u!h{O z=sf)~Rl(Iyf3k?w?+wuPZ)834uSEXYqzOAk*arGVy5O&{+QzHs+8DT4)Ti^fn4{qK z9C@OM?AeK21UDcT1KZ(ZtkYlhTZhAbl}+ZS6PLJQ*c9AzF=@T$EwD$FgcvthSu8bv z+!4d>3pkVLCYOeKgHdZxYmElA!Kl`0^+t_etJZ4OI-Sm_)oYD z-~u21oCH4HLOG5yj?c9)PA!aW)sW7ot5ys-D*sbVQGD8$dlKwZE$q;*8W#0~=u*t- zuNyScBiW}G9huiOi&MU|D7+`I9?KK;8U$H6r(cX^(3<5 zo7u8m-ax_|w5cC_<&}dHAkl?Ecy0V&M-Ia{*fz8-? zsz3!kn#r>iwGuJm!*cP959Y`zU6&}f(r+Gkub6>(-kBakl8%abBM_l2a0Ls`h&VyEV_?Y4r7%2Rr4U=-yk`Dvf6%4R80vCndWK(N3YNIGvtvX$W zmDg%DnhH&Y@)~aCx`zl^xo#0Pr891`nbPNUz>e|cOeer1f+qaY+w%xu(KWt3dzHP>{jO|;>EwFuKw`AGAt;c7x zZQl}CPGtLr59M|H#wZrX_N|!?c7C^SjAWN=-!@=x3rgi7$N5)zk#o6L&~?wT!?_hs zHw0Xn1>R`lExNY5H)@aK$>cSwCZB^lIf29$2A_q^5#6FKibbACLR;gz^39)M!1|^UvY_x8V!ykpb$qUYZ zFk_SV;p>S_-qtN|znHSgi+}yy{P|LV6=?oPg$R{%7W95I&^r zW3f0rHoq$;+d((CC$c61{3heqOUqDtacn^MXi2I_$q8ubG`@tPC zY8YFfV=$E2?6Tp?t{q7>uANKw`@u%+@ZbdHp;8bkiz_snsn?cqGp0&}C`5@1*j{(ihJG;w z`K|0t*0kbC1ePE_qrp%~Z5UGjQ)A>914*xnAn0|1QExD)HGjI4n5R zsmWD1)A3}-Bdy==mF=Diyr=;yXFr%7wPZzne$z+tG(-J9vtpZQlrE_AN&3_iu;;NW z%pUq{eBMN6?!5K2cXm&I^YMgXv z0aQPI*PHvK=@2HRqaUSBl`nn;oxmv>Chu40$y`g*W&F`QSc!Ie%lxPi%rGY1K z5b&CYgcrUiQ=*~Djmc{|awz4Rp2XPT;d3y3vsT(}ETXIlHa#9(_9|_iO^;F50Gl4C zu|B2vg%UPX7dP#wp0FEA*h9*mG!|IZGY6*9CyKcEoU=(BIhDz+RiGj$_4(~)v&F*G zKTJVe;Junu_eX1m)9iK&YWnX_mnLuX*Fvy;$~leN1AAr-#0n>GO0>lSPzR=vv5O9C zt;=s=VY$bC<}wH~9_sava@AHk-Ifs$t?UG+%by{Ny3As@>@r!At}GJN*~4By1wu$S z{7sh_oLUVAo7yUxAlQ%xt>@5N_28;8m?}gO%)KVPDbroEguXP&g3?yVasIW{^QBw} zwAFHU=tCpky@S-!LO=RJL%W$1+aI~7A#3IY)n5h@e{yltbmrLZNcKc8=6exHV@kfn zF|sF6OECvAIBVKZoA~&yWZFa|*&|6gMc(dk+C=o@744=?1a`Geo4^6WRenm^M7R;Z zW*R9iZ6eA>lxd_6rA@FWa%<kE?! zX?%WBf1>>ox1?s z+$CW|o<-Jk9)SD5TyW}a1%W^FAZjnls9u=(Bcnk8L4m^W-Pq7uvg z{b_cQqp40OAZ;1BPEy-v0^HjPa1X4)U%kr$+>v@$2Dl>wT@fLHgbO%A0{QRq2npn( z+eAnpH-%(A*nk}8-+a*eU)&rpAAHFU{jPMn(dgqHZWXINZpn-gYr6LsW(4LXWD%b9 zuZ+AvKN}>cU`1PNChq=O$R*FqR)grwXLf@X5}k%%MVy^H1I&dd@eGZhIx_psr*}#> z9?y30=$u&6u$ktU3KQ9Wv z`Wd-}Jbu~(rCwV6wgFzCAO}}_Tz03sa-_#L-j|jrG(a`06BB*_bm72$09`n+0YMic zOOdlYfi6TkF-d2#f(yI?>BoU4Og|1hfgPb+1!ZhG3CyD2Suf zDf2KXXP`AipTfSMc7}q$v)J!KG!(?2L)LR{D7cXe-+eg=-@SS+Y+3^ULZaCXV<)m< z$uJK{?VkGPf#CDqSkQ849*DeHu!0s|l^;Q?;F@<|OPY7TN(C*#i2fpJfd&w-^3gJo z7p7R0dtu2qK|Beulc=2fVi#R~dWv0iUBQZ7WJA8ht`Wqpk%?WD5zVpKNmNdKv5UQO zdWv0OQ^AT|JenV|tKkkzK8GAwb^l`&0cJB29h98ym^s8(mSG>E>0szA0;`G85i}TZ zOTm7QBEn>Pq$HZBz7PhtoSs4$d9z@JFuXlKLTKR{zB!vT{0oIJ!ifH?tq?L{L^!6C zlR1gRsV{TUw@)9LOG7@}nPZvDA@bR-{K(uGu5s;^q;c&!w5P-v(WJ~tAr~j-{|&Ha z&-j1a@&5*Q^X3U97+wb z&H%z62yCILEi_J?dP?b02Z^KSz)>&dK$iArC})#r?AS7Yc3{d)l7m=;JQj*_7JG0< z&;B$)9dbrn`ml5qJ~asj-M;|N|L8}P`>;e9N;mL8Pj&+6V|YT-ECCQm%DeoAC2>NO zc`Tlaffi`OY)W)DQ%Im^cS>pS0L~xSj2)J|#NjQ$rT`0r$8UHNZk*fn1xxHfmoB8h zq&%S^(JvchzcM}nX)y0REsqy|qa)E3i4nv$V3b%Jct-O^aX|5IV;hQ4K%g0UWHIQm zg>N&wAPU_dXFnH_Bd1 zlD%sgy^%Rc#hDJtgLnM1h;01gZHi_Y#xYZtq1*aKuhj|aqJ94H=tP^pq zGm`N((4VfL(pwnE`PWac5$*p34?6?H;eEdR4T^{P5IDt|$xLjCvGHO`B`vWf zkhqeJOliL!2zw&8VJR7jEfUFYn%ELqNyoh46I%kSu^XL{*b;sjo67FL?+C!yO9@;%rPumTy#deP!8m)p-L?>1UCC-}cg6+;a z>`UQpch)2SUA^7efLwH!?GEFnkhVLFfc7LahbuB=&t)@v>l9q6_9|SVRyrh>NC7sT2)Acf@iQ_i5#KCaSdT0 zrR)(SiLjBwn*h}LEqRb=g|TPfqEEnx!8S!|3{GH0(KdNMfo+qPyw4uWVeahE>GZ%NFOM^C~S~LkM#Cfqz|gBz@mVF3RRvu4T2;iQZ%%U4zaQ;BSnMw zQguP3Xp9j){USwo;%{D%k)peh^_)kFirmI&cPPllX;g2P1Y$__5=nk3TyCoNFO#Q% z6op<#V>FYoSx?u6Y z4+$6U;{OBWzpEGj`;m+8viN7*6w>0Kah(6f|BSP_w_x#q^ejA_ech-luF{V6I-#JS zl;;+?tG;PwMj;nwPb`w~Fj=PC$ro|KH9b#etq;xv{7N(e3M9Dl-?9}CSNY$&oXgQq z0r-+G%fYf(#1{04QZ6ZdY>NdP-2S-)nF+{Ufw_2w+k%u*F3DQ}vWwl)7P2ci6K{jH zJ*@iEO1Z>m$XAkf#1!X5_EIkFq1>7h(@VKX6uT=Ha2?ljw3M_QeG|MsB{3Xq<#}>7 zCnytXXEj}}&VftiuUw$%Dk#jfeJ)^N24~S0^N(P|3ves<_0Lo~QkXTSb z7$k!VgThmlqQmQ~tYRgW^$wlds<%~Wtqxw#^P;{&;B`g~RCTeb)moihd5eN;3|&PU zV@N(Q8_DyD-B}uWwv-#|g{tO&0h+*Jv^ksuRI1KkCc~@4m?0g=kSXVb>1dBXYnddDN0F00rwfY!V1v<%qS z6x{NKz`*h;$UrzuM^NcP27c@ISsehqNa<&_^CSu&((;B9cS+<|4Q~QCVXYvjp{}xC z!|T<0t&Z2300fp7OoG}38cuIiLy4SD%;>#_TXyu{=aXee_pFBUCQo2*vbkUHG{YP8@wX1C$X(xVXPp~icZ*cP%>d`_d3Mkm&R9zj{m=2di5S7DbJp=+}eV!vHSMpZ(gvmy-Lp$-(C7P%eG z@0?F|G`|ZkWBh6CEz7dVe1S$f-Q%Tkgm5I5U+A}v7gZZO(Ai2Z)Bv@uLk+aHb@Uk| zGdOEs*%E>Nc!@IFm7@Pft8ct2-1GSFiIZ!*1qe0X!s>M+q3=qdfSOuX%x zcvs;Vb(M*C4N~ty6YpANAP+h-Gn{=L5-wa5?|S6Ft4+Kckc;kO;$_?vl3{^yoPWc@ z(|_Ua1;fH)>>wEy*8_6QL!w?ci}QF`(IGo8S{XinW!1m$dVHQz3SzIo zP?7Xf-Dub42`z6V)dZ3XHrzkI7g_>%RwEZm*ivJymb{m&boB0_=Ta*jwX|Em`@L7e zN`R5`CIWpxBSOOBo6_C37T;*&>#~6XEWVrZ3pFF0!r< ziOiS7hxLHLfht{M9LyOH|*E%#dqn_8; z)OI}-{@`_LjZv+9RKdmea3r>9HtkV(1CA!G`0DjIp-#t8i8ifk?DG$G+lSJv?s0mS z!f&Kl=v;m`LGht|zb9)R{s^t_uOPqv1i#mbcBhqMQQORBy9i}MYQ-CB{WZ1z%WEAD zuz@G+Q?$}Cz)&CWJo7eI$(epTKy|QF?D-&i%^Sfiv8-j)-g8TWuVRNrOC7RQ)BZpm zXSr8U#H4MfvRdLYsyYMYV3F|iHm?qAf1Dd@%a;l|1Na=YTB}vB*V=i5%_!Op22rCm zIG{>Ng%)Z#*Uz2yLsCC?`pXa%yb*cLW=q2wU)5r@!)_ku^bfLGZB=5aiWOk74}h?d z08!ex##zu-<1E@%qz*t z*vY2?|HrmqzjL>f&mHi8^i5<0I2zdAHyu;ao1IQYJFydNn2qv@K93e zvP{4giPotP{t$nw8w+0!G5(_O6s+(?cIHR;>|8WlOrlW~z9=JVJaOR@Q3yLKcoLmc zU+`k@ot}ahdcR-=FSs{9f;X1i_$h=YKR)e#nDs)8=&z0`cyzC1ts6#{k>%NCO=DGC zvrTeR)2F_yh4-DFvKIKTU}Y_SAV0G9zg+X@3ex-}%39!KBr0@B)@)EB)X&zLH#jn< zw_}N%MBx+^xq}cI)A|mC#_U)^V{?L^;84y}(CXT(q7~Q&L~o5(guiCRfW_SJ^4XXwtKBp4N}6mzjX|`^)(AH%XbJ`$W+h5BTKK9U zuw-BbyLhZlc zzn+>8>&gftdi60S&t|Rh*LubBVeB{t!sz^BWgUdgRo7VE)QHYzz%gQFHJD>E+*vhqI0Mc`)^l#voXCZnib(iFHUl!E*B(<2+5V0K zNmqgdQn_} zaakb*upd+hYO3iWp_msWZ75frIE;5o0wdGt;!MtfdYFnlDYaUaGA{$IqD#D7g6H_W-l`@GfXQ z{493m>a1NE0|77gT9;^Vo2FfsxR${}v;+Nfa?9pG+V-+JENvS(_#SE706TG9B!P^y zZ6uYiv~47o|6uRw&@)p46+>B>!u{k5_@j{ja?NH*%2_$HQ3EDN> z#wV3ziPa2})0G2A&N?o*YR0)FIQjW6_MBT1VI(iGRa?C^7DD% zo6TkdqlI127y*z|r&BvL$_xn14cOa)QhCU6{#D+F^SEb0<*jCieo$}G?-}W-yxeLV zU+GP#JPA1BD63}%xRI@I-2pm;!C(@Aji5zI=uJ%hY4M7VaHtIRcxE7dB{lbBJes6J zVo+p}Aw}>!Q)>dlA%^OOkJroi7!oc;Z2g0PQ2`JYI&wI^dND17h6u)b5?S%hYyil_ zGr)$}xSf_E#4}n`@v9&(OO+IFl0hj@W$~y51wmpRu_QDkzW;8SE@DXlAO-6#o&g%> z+-}Zz+w?=JT8}|1`+SL9EkD^1s4wXB4K1%rYj^8{AGj%%Eb+C zo2Y__`+=n_EuZ~Gm|F&w@Y6Q=?o=805E;P+8SNUa8h{Y&x(Z(HunL0RCfFQ0t0)+a zf=;C?suv}?fa{JEhV%aHvrW(%E@v{~yklz;ZOhPFZQ#5E zujBJ8E54405iS~`}sZm)h7-!z8y*B z$zT0863fHlJp-I9&Iq4={_34L;05Wg-i55^*IylDH1p@LW(3dptK0Ke@5UdctNhh_ zka`#TtM?)UdC-}e{_1^5xNr>~A0Yo-ZSdHSTyz(M2jixY3?7W*{2M%;tmCGF!DA^q z^z&IM9;HtPt3)>o^{&81eG*SXw;j0eNI(2iAU$FCPklD)zo|!fZ#dDSD+Ama|LETr zPIcrhiUQsbKI+MXYt~ME_>;Hq0tZm0>zdwRBr|v3`r13Yr@#3)LoPS5vX5%1>?62# z(A_MMdp@7Lp*K6pz#Kkr10`sgcSy$$dS)Rpkmx&K26jmH5+eyl@JVdP zMB*>u8MB2P+&cGJFsv{SPc(4_Y$2`kmEszop9DgI0FDB9c*u-EAlO&=fr4wjK)S7j zJ*wodfdWhxr^muTNG;OuYmv;~mg~{UF$RBii_L5GRjIT>?O*_&who43gMKqi~Uts10175V2G)p>%lM!w^5NjMubD!Ln9B zJ=3QqQNUtC|a~TUbm&4}XT0pm!nz=ODtYrAlSAq5HVy@g6}B!+z&+_q1H%F3=6eg3lQR8f{Y*TJr(Z|x*nUADCg?#8}$ z&jn@i>9{lM@T)NQ`_&k`Md8vkIdg)zY3y&KSuf&%Woh=7)!yhhWGbed6{>7RBpQv4wH zyAWrj*w@H<&S#|GaIH^2pdhVJN2XG%IHP)uVEYcN#sUlbW)?e>+B^-+8Hc;El;zN{ z7;Y_CDGMFVkCgq6i!}d@L>iCI0U--Bq8G`NF1YQ=WdpvHgHjDP{SHW5Op$5LklH@8 ziJyqWh_RfBLn$cjC`ixW;Lx*kXg6fK|QbG8A$(jMWRlw zTyKZ^Us;i!OIgvLQ65Mv6uo1de-;NAr9wVqY*6UgenZ7L(dHj&vxz?6P#5^R>~%8a zRHCGFqWyuT**~V0AReF!_3S+B$5z1Snf`eZY2F*b!aZ}a&weZ{#JVW?n^jrU6tnO< z)a$j@T{56lRc^7=x}7(d^Lm{@r_l+3$geZ$bsD`%4d1AZyiQQ-HF^QcOzRpIhsUHu zaELR26HK=hE3~3b?cnXa%^>IvIuma1xu))9sHxPt z!tr_7@7%4ab5~b5b|9**t@sVNB7zd9a#$8aen4nKq7o*Es&-@{>sH>0N`b89|B$m?#5T5 zi`D|cdy#rR*8>0G=D!o@P3FI|_?A8;@%xbee;(`A{6k%($B%DjIbD&eru|g-H48PX zcKU3n&@u2V^?9n)?N-+W=Xh3pUWy-sN_qZrkJYbCUyMZVM-KA07-@kptkn<(7FdX3 zV8I8F{tN}MZ8l$Xlhsu#rY=XuI{j5QS-n&oFSU%T#tMQXhF@}WYmr1V1515=yV-29 zK**oV>GQ)Hq<7~{l+P--mhf%Gq$Ri$BhEtDRp<@v>GdO6bwsn7s+lnu^mdcSX=fVf zrnF^=&1LoZp!#9T0JP=2p%bV0yI{rm#!xcrjTsaGBDw$>H%Tj{>>R5ss-5_#IWnUoy z*HPhg(=bv?jn_k$)_U#br3RbLsxelW92y;O(hJs#3cEqWR|o)tA@B~PLFwZTJ=&if zdh~S$I)p8sZGjiM!Ul_6r(ZRx^TA3kxw{Brkooib{1L|3HvUf~0gQB328w=KSLW;i z|C}Qm4D{Gz>Tl^+^z@B?;1}tC?A%}QdxiA%mdZU>*23p=rSq%%v+G1UpXq*R$q#(_??q_^jKbV4!gc}!7KA7uzUW89^~t#v3mTzxAc6Gbc{;R4eZCBm%i>Xs&;to zL}<4(Zmyx<1SUihy%_?}|W;B|@rmCvrl?=N|#h(=WF&p+OR+xDwp!GgOa zijmeZaN&!8W1_|qj>w!jg9{4*uR_*;AKDY9Ec76Du?8oKO z^>8`6e)V8>l+TZKl<&(tE|S_)^Q@iNos91um;M3yR!_STk3?YKkCKjYo}CYw%&tdX zVCR>-gacuHEEct4zV#&c9ik2KH?lHG-k0Etmz&$F(56}N~uk*b_ zi+kMYoKiLEEX75_uN7yFzqqK!jTb3;+^iyj(Ld_-VE-~`_aNN=;~qnv8*)xy^zPyx zHoaf`qf36#BQSbT@kLF0;s3tki!S*P?z^aG5B~q4y%?Myyy0@7J9NpBzr5TSDYm@z zYktVht9taf@&C^6F*o&^sz;9tK0Ny1Xi7T(&t4P#uQ#vhHRSL$=M8yoeD5L8SrliF zzgW@Z#$OM(&|3<}svd#!tBO>UK7jsFI+xszBWNrM1Zadlj0o-Z>HP1K#W*6S^Dlr& z!s9mkY_)cexyI`Re6+ehl>}+-Lbh@GiDDV(qdhB9#txMVRZ>%boAPazkFp|5z4YUh zlULza`LCn3=cRmg!XIl&O}9$Qjy)Uxb_-Rm=an*LRjN!^YBB*?xyk49k&TnUA7h-N zG?fF#=_KhKnt8rgrisTOd?%8(b2n@Oa!GPNk1>j`0++P@-j{yqaJF-<0bXj)jyBWd27_g zZ0KvGpJ3V_zmex9<1JJ;c0(d&H`rDQlCPoLp> z@Z>i*+DDHLACk^Lyf|95I=z69P9Od`t z&<}>=`UEe+AJNBxyS#OSF;3cPAB?$5VFi0nhHqr=p?vQDkez3~2lBc}eqT1-`6&K0 zG95akssC!K*g8ype-iIs>P;!1W&UgA_fI<7%hslfm?=3lQf5}5=+9TMrI*~A+*CrV zo#I1x$@Jp0id^3FyC&VNB#QZT@QNOp+GB2L>P3fkpWWkd2-ZFql@`Bg{pjc@cr${R z_qcHGv>rX?AMMlQ@cuqS4yzPHOkA%X=cz#4kNoBGB}ZTGF_*4^jt*07KJu5Iuntqq zy&BdzEyX5 z`OEMnzzgzJLHT~U2k-^!!<0V054_0zym-&iQMmc>)g<50Fs2>C$iy#V!seIs6_Ms+ zS*T9+HyGd08rzvSw2R+RwPFa}pW;8o6vf>aa8bxv4zk19!#sA;#sao5PgzY?tlt=f zB-);IPcLOFmn5>4m;H-^zQUz8*~(9DqS&x-MTTU}OjM>|wMHUh!>0Ze*czZ`Tl%ov zVLmm3nPOG*0=a*FudJEMj1t_UBcF$#MuuoIFs~=E_-BdSWXM!*LR440Mp0;&2PDun z-UM09*hx%JR)c3S)b!PDaEAi95qJodPJAOJEYE7jC-6)|q%+KNPGl`(YkIOG`7C18 z0w;U!j-Jn&Hl|+2#!Mv2Mx5m^#}>X#`@nLT!-*@9!#s`kCmTxb&D^vmgLFd)dq~;W z4ci=_OB8EVJv-m%qjYmS7M776$gNDWcP*ngG6(c-@5wbk(~~rRwdiZGL&kAnl#7b3 zPP@x&PFU{1yl0{gI^I&_2TNgRP?gM!14|hYZjy{=A~fF$6%l*0I4 zmN4)FLtr5G*sRoAnY_Qo>P$V&A(xk3cG)jexv8%8i<8p6v3cBfCmmg|qkEmS?9?v@ zsD3A?2m9)Lez7{~V+LDcrReZFE35n#I-Hi2UkaViHVpt1^J;a4u0pHRn$&_uG?=tt z7<}&y> zDT}2lKxa-qe&$6>6Nm5jqNV;GUJ>M81Z-@}L@!v;#Z(D7GuWAfou*^j9XcxU9lpD* zHh>>ig%V0ZzjL7#GAj|FRwyJe8>Ywd4`%cBdE81^7;a&Qeq(CA4y#Rs((1OWyq;RQ z_)20WiK|6giQvu8U|DtiN(8W=fu-TBDR0r+`@xAJb;-vThaIg<5RYe8Ciw8xL}h}_ zM|VHb;mQQ}eg)jnRa}QE69n(Rr%h!7>^?5!Xk`NI9j^N1WiT!gvsEU*2icl0qcVYf z<%B8|(1*IBG6B-<4wVUzk%FmAfZTpx#NLd`1oWlLRb>KfyR$13V59A&^?Z-Y1lZ;= zk|n-RWr8*x@0`j6*lZpmBr}2fW+37J!GM#YUOx*hF`Gvq&=QOAWt$&}ElI@ULTFsZ z=|C2c!-8|*T1q^{0M+(R3_F<$P({<9EF$%L0~jzf6Z!9IpuXA2MRx(}W84(d+JBT;131NmY9a*t&v0U$tE;GQ))v`O)%oW-BZo=qA&0N9zVM&9nnTVbt>8WM2 zR!TS9R+fwZ^C5T=moiXt=`%b-d_EnNj>4~jbjnRoTaMk-wzgbgA3n`iHwPnIk`}Gl zUir!iR48B%<+egW6V#)Q{G7y**%FCjcdac~$F&?SB`rtagn=!I;$SP!tMOxPHw)c# zu_AL<=fDp4H?H}fKBW2WnG_K%@B#fbl_d%L+fZgoHk)aYA=LoYp(5duc1=GHPpuo` zR>5F|Mjze*mL3hSw;AkOhsI{q^BSAlu7{iwNTJXe)yj!n^Wk2k`ME<-%Bv*Ug5-aZ zAkU^Su=%i|pv=g-%|nW0WF15cF&M=YD1^0G!L3=@o2+R?%7Ohzc{H>{+mKC6DQOmL zlbcKt5d^(XFzO8kwMH?i*KR|WC$?=)7|7Cm@(j37! z&i@>7%URqkm?NgLL%*TJFCTQx@Uf#xhk4v?(Iz=ZV*6Hpv6pRkc>r;xP7)z)f}qoh zj@-HQ&ZaM!LxH(OcJF!msp2-+t!&pj0%sZ%xmI?HOdrDh9z2;G-TCTSsj~#axN6AP ztuK5l!MJM4o;@$T+FL5ixAnGN52S8>mpyW*h3RzOK!Tg8zCcyZ)0yI5pFmfn%`fVA zHK={;AD;1K*N%nsacQcUJcNDH5cnid6AK#Y9nv(Rkf({}H+DTP!NLj#68dVM3=4~o z0qqnrfZ_UYB+6!sIAan{d^IGRxB|1qnkf2nfE&bcEv2l}5`7+SP1GP=En!b6`&cYB zv^ZW)Tpup(z~)kN02!4D_C#v`lLLkOFicDvilHjvA{gnC&>lKg%1f>G3efZLVoIJ2 zdj)$>=_!pM2M-8@KGOnxs04A{FD>u~sN%ha!EFO8;hv;a>7L5}c#Ye@tPR zsdCifUmgBsD`^92J0q!(v|eAP>p=C)|7b@sTe$MD;AAV~kG^R@C~8UXEq!zK+9G*{ zTq5Z_8AF@u7P>lK_wGIW6FM(*b$rX_HwkTn(B?Rciy1i}~-GytfL zw`xi7j(ts2*&v`&(~-#M(R+cgx#XlQnkfLBgsjRW&8T5O2Qlv-WVWF5&upfz zj(w~ZQV&vfknHpX3&Q|aV^QZN+N^_R+P#qk@TQIQ_(W9lzaC+@X4C_fS1;H@Ftt{b}}gAq4;uS^5vI z7ONe0^EjuUrMH!;nDf;>z!(Swh%zHPnKju3wzqZIm%_EZtw;X5+V-{qx#%voH^xmN zEhQMo`Cm$W+LwC*mJ%)O(2v%ZA4K=e0FgduqFmv8PU}3`$d$0a(Vde(z5|aVt7@*1 zRuuGmomBVx_yK1XFohRQGEsvb8pYJGw!qZ{FEx$d3Pi(pLK=B>AsOOyL|_}HnS zRM-}(eT6qOFgZ9U5Q%V!ja6HP*N<+Ec)7q3Rv1FZ!?8ka575#>^cfuqFMKbx7*PAZ zhvrsb8WpOET*6eBn#^K-!F>@#)+YO&MGs=HW!utbJd^Yw9ug&iq&C|sz~EYZD2oD& zuaU2ufC6L>Gl6uYYe{0Fya^2KD)WSVAM62rk({y5F^K$CMJ1{XGb(-b+-hjSF% ziVx0(DD_3pf&(J7j{c}h8O8|gO9@#so3V^6+lrXYesqrDZUi=J=Vr;|_c>}@wUth{ z#b%>H@o4liI1GZ-YBXuoTHb1OXmxtK!7f-$CK~`gnM7@dKcX4A=?Z_u2Bh5m{Sl1M zf|^8dWb;3XeAJ(N5hjs6?C^sd#ydTwV<}Xx-BT&eBcb<0Z47a29x0ONk?@}NcPI) zi5C$7RoQ)~)K(3f?ZJ{%L6M_O6hMjO85F>FPyj=#@LlLC6u>n|J)bCmnO+KHpewu- zNVtH#6v%&!p9RW2(a_2A1OXq?*(`UN^iv&>sN+C^QwL7FI3r^7fZ%4!aQt%7ZIng&6PIsnn z3TXw;IL`kHe)4~Ex4;Vin_uGL+^a=v&5c&y_~BNu+T)fTCL8XQfu7hR9wL{S!{jW< zVbZep`41s0;&`#kG|j_ATACBhVMvi2#EedjY>-yRbfLVKaG})in*U6PT_`K>jU`73jWiY~s{;aJGGFe2&-pR?FMB>z!x#-)ckIXf{*icNGANYuZ&;)koSmwe% z>byiaKeyr&m=9KL!ucS+D?c(fhHG4VC23r{ZY~|K7$cgLIVmp;<~dkcTZnCeGGABwV;&^be5# zuJ)quM=rXH7oBlaNH#deasF*^_xwNZF0jGPV~2hs)X}r~h5-l#7TA-FaIKjbqVdNc ziUWuUGsF=Dk=IE1easF=ujmLTp$O1317z{hqmgtJGc+2#Q%3kYi#o!!B$iBa32k^W zHOVEi;v0Y$VtIcC65hBy4L0SQrQ0%{>F5;aF|SY%B-YUgry;?{1YNY9X^GTb$;uYk zjGd?UL*Y1>p}{AmTcPj=Ny7{cvJqs~wOJbxCnd#&-dj%YtL0{GG!S+}34192n4uEg zuEz|G<9I3y1T!@F6#ey;*}{(JD2^Ybn}!)0S&nTfCEzSt`uIA4n+HU8%$X^7iUp9B zI|K!I#)}rfy7xmNcAvktLKWyljEIcL=QyL^J?)V>jC>Bffa9J!dt~k+pW`nf>p8ba zj^*auH6Jir?s{e_-IztB7nK#frQ+WP-L?aD&I1ekW=1$mEuYz&wYN80g`?k9d$To2 zy$kndYmtF2*qg0G!iBpxTaWyA_18+V_GZtlPXt7Kex!L(NaM&uo7+S?-;upp=sO%I7Wt62Hw%3$UxCPnj_%FaL;2sENp!n@ zZx%R8N3BryW+BFHr|iw50s7MAs-6S3-MM?S=oDZZo!Y|ykLa4}>`txtu88q_2* z?Y|is22X&#oSik00e5T2ZG5oJ+dbOOq&VD9bFIjani~K zmmd&Jx*0owXFQp;4$p8o!!YRpjyClcuq?A4qvcsgT34;zJf9)Z;zLp~2m?|Pj4ZDU zpf}h_GahZMLFN> z1Ru@Z?A(&v>@1kR>Y)yAb{g(KkWa+{((MkL9b}|H98sa8$ZaQWcIZo&tIZC!-PxNRY_yX%JJ@C?Zg$#qymK}?*lbs9 zc93uZZ+4LXF5m1R7u{yFgWMF-T4pwKod2~90BA3RMGOFF)59?UwC6*Oomr_hIRNjP zf>9I>fcD29w-2D5(!(rB6-%06?IN#h=HeM{dXfOz!Q?FzsRGa1;>Sj2;sv&}Ne9qQ zTuElIrR;`|0icZ!yd-EBAol^#behLKdwTTPI&R0jd@P)~s(HYfRtqzKW>IbN%^%P3l zKp|26h;Ze4n^rK{>{f$at+(rWqg7+KX>7bz6nTIwH|g7ybilWvkY=O}$Z`H>q<1-P z8q7#r+2PzV&Pun{RXU8~af?~(iPW{g<6I2VWjXnB5`bS8ioBAz^h@(n511PTK|@5DJr!z$l_ z@8j;d^6ZRLlTN-3&-5g;GQlep2aT{M@ET<#dCf$^A7JxrTAH?v7G0sf9ViSMN?d_y zsrjj62E2pq{wafWLkW9Gd7i}r0lGeCWwKHMmxDqDH*Wzv0tDn&liYgIKl zAPL189{N`iSu}MvO;?QL7%4AAya@KfO7AOTMaY#m!R!4W*>mwAy_}Uvrb*^dIi^&s z(26#-gSYcGgP?~32)x~x7U-A(UbCP6dxUPy{tHw3xaxif*sDSaMF3iEbvAX zZ;{JxrsU1gojp_sUbAZQIq2*ZNNnMwy8yR_g9{omXAdjLnXYZ9UIFA5`Y-nG^3@Fa z#z=mSEdW5{0&D>_0ys}o&Ug}HXNYSfkSHYZ+#ZKN3V9tI{cN5bNJVu4ua%0&O#b@d z$`p4-{Oj-L&rjslL=J9E%d5HjbI7Yf@+J~^0-2G3Yyxv`Z~Bzg%R%=9Prk;gV0am{ zko-@oKMd+H<#8>Um3$llYjlJ$F6G-`I+FB_G(NOJs)!*c$4K7Yzx{2K2x6XQHiH{*&f+axCHPnL;VZu>hwA=|&%jXXSpJY1X@ z2ug|3%&1UPa*}j#s(*`8$$WZ=qy&q$x78>t1=$jdL6i za~yuiRxT$vWof(Ra)N8yl*@VIARQfSF{NBi@ZVBrXhv#U?dmWaPBs&=>eO`mg5@Sl zlha^Itx5JoZjDA6cyAKP?t=H0QM)>hgIkE@axyUD!u#o4qRcNwTt;dpN}cj&u#>Ue zTcu~ z0@SMdI}o)h{&m5MVC^+0&I>YvXvTKOQz^k2;)HRf$9p}@ z7(P`?2LM9SGn--!-hr13jqR9!6IW`em{v_b8S+i%X1MOG`f(vly zyU02aT!hr~X&p%4tj8B419{l2XTn`SjD!m}Gw%`PzpFFzmLM11B{Pq4Q&6D?IDYvL zJy?1zcMpUfEMkX#V??*#TX)4I(N+spV8=+I2L~D+NP6aJ*j}RMT#*!d(DLGk+aPrv{p}saEW%gK4m=AF7RXmD435{3diEs7A!tdmaAWzzZfpZBlD>dZQZpleCg<~3ZX@g4<%Kl7;2 zGpcIbqSX4Gu5zf&Zmab|-jctZ7GL+3kA%NtYHMmd5c5y<0XmRQq*_l#B|3@Zsc$9- zyxNV0u#@fNsqqDoO*t0A+@Y_BH|IwPg9{TGwpDpOZW`8F23S^hr{C#u-++^Ixt030PF6}KmQH;sjcz%8q%_r-7kewmQkpws zUi9t!Na+j(*ZAq33Iczql*Sm@E3-=J7)blH*SbVITe{OU4td{KHe7UB>qPrlk9Rz) zrsB7b7f)gal^8koWi$TH=_8w|p62L#IhM`bg?>bK=S?=3eE_lvf2eFm8QHFvO^K0H zUp8a!pFXmgI&TK{2g->iih{IL9)WyZJF1&yNh=%!Q}ltsw9Rm#&N|-%%NaKrFgi6}=}R zg)v6+)R)5eXQ!tWM!zgrDU2M-j}&^j=J#PHgg>}QWmJU@N?{oj!4r)Z$uPG@8PQW; z3}c5+A2Cc@kRHjg80K(6dNe;`=;C%%ksU3VQ5gtON3-$*6hfEPU0I$Il+`Kulo`of z62UX!TJ42vO~xOktHQOWA@wc{*P4zDhy37AoVqe3LQwjVP*S3>M1>J3n?6IS|Z0~LCDrzJj;L- zPPv5wspBneA%!C|@dCtQL4;NYkh*;31R#Y&*j#QSv@(IzDVxgf3Mrfcq>j9DKC#+D z-N7nX4$vj*xZoR`=6l3fF&j-(4ps8QoQ1q7``i(&}y(=a_ zxQ}nNXpH0-_~z$1bSvF%Q-gT)%{+&Wl` zkkQVogSEI#$2+AC7HhT(wr>w(e+qZ|_6YLd)!Vlv$VGSAzA3 zYzDj5p|KhDyvC-s>qV=b*Qqr|wK5H@I@pZl^MY2LfeyU^DR+N#XvSwDO&W~j{7)LU z_vPlnq%n&fegNQ44yQ{j9b*NYd1=aMnZH!dE@V>%BxuqP0@=>WD9Gi+QtQQ^3pW_Z z(eI9Y`}L>0Uit^lF-B5>O{P_j$9<`q%^Ezpkd==Y$bIvBWt|R|b040)}+@M6vzzs9a`-mTqD9;>-f2TT;%TbjwU^hs_nsr@ek>B0)2h zxRQ0Q=uJto8VM=j(Cney&J}knxbUI?1koV3B@#(JCP*fdE^e}Voz@B$AlHutJKNPB zm)+^Eq&ZVQRZ1%7B?DBmIx*D|@<&=e++0LjUJ23E&EPCLw#vasPvcL`Bz9wmE1$1kQT9nlruhBtbv??Q3;haXU>DNye3pqwpDDZ zMq(@1*gq3b$}3lT6ElqQiBxeav#?T)Hb3^XWbTJ$YIDk&mehZa*Qg!r#DMDOhm+~z zDeRyoQ8SYqm>2_r6+Ky+xWLlH@w$d79BndB{BfB`#8d=xtnN+xvt>w*8yh<4~}azsedHmOTkq7486lE1>Q-Zv`sNTPc+&qcWl$ zDji8OyxwCM%PS~XbVlnxxLT}s*v;dd{y{b?Qs5HyV~l8?SuVtq_n65lpI{-RXU`vVrLnp@~oXrqm)J` z)&V+GVN_SqQC)>yI*47mHY*|a+jV4AQ&Trphl-Csu7Bt1m%a;Vz)N?)PaFCYndqeK z7gciyM&WOeMh^5tjY$8iPNiRS{O(tBp$1C6fs&6h@+X>MpbGaqJ|QR9cnc6}yoC~q ze~tY-@i1!DiVh6vI4&|1NJVB+Qn5ow>U+eBssbILm5x&pI*i2fV8G76sfaVerv*+$ zddwycc zWtSwiaX@igJR}Npw_g=^$>Ef$xSt^xl2JFhA$WpSaZ?`GlI_XIv8>-oSH)!m%`Cp@ zRdHj;He3NyOLIr7;^IU3tcuGB=CLX+4p=({Y4cJQcOU(2+F>FeRdGK+9`arlmyykj zpE084}DX^~85O(JfdsNlQK%f}u1zyrw6! z*p1Bp5Px!U)AZW|f#s~Ou9399_3?KttQixTvkgzA5Sss!LujJk-QF~lJtR3MnmXHc3 z8hvyi0lHSYkIj=Ujo1ZfPlm2dRF4O%N=x&-#f05b!X8t0Ppe9>6Mc@)DY#Tvi0B?V zkjmUyRdNRl5%~aF&$(3vTmlE@f=io;Oi7fHz0v6@_aRnwxg2k7^LWK_r`s=jp?Y>X zB7-tC>npw1YVo=Z^+7>f@WQZ~+Tpcc4D=Cxd?_tXT379<^_dNNU4_YEQmfUMW!4*& zcsLCb)KM*VQ?cx1JkV6J3>_#~u?&8iAF*7i;C9r5SngOsXVsDrBYT_Tq~+3Ot*)?J z%iQ9`GNH`r8N{S~kR;`UsFV+qrF;;T@xRr_1AJF8a*c^GWoXb*sV6{vFac$xFlu zPH|#+>N|Hsg-`TC811oMMOC~5p6S1Hfv13E;3-Nnd=yDuDU+=9T5GCc-&%gP)oY(< z^@^iKw_Ws#-UMlR!t?l@j=JGizcnEUWolKl1N3JqxImaP5T*Q$aILBXF%Ey{wm%AtY=4w85@C#7)nP`C zsVUSHP)arF_@NKK=jQLe`#ds#cm1lKC4tE}5VA>*<|9g}bLCaQlS>?Cb-6065EV47 z1B3hrE;Rdo2;Z3f0E9Y3SktMgj&jp>r?2M!v-c)&l2+CI_ovmwHX5kpO$;%{v=j9| zpfgkZ(uw{>V$=xgtH$ha`&89aGY!4j%K&~7Q|!IJ50gU>K%eB}0;$;x4;+;iO&la;yM zHe7iiOxr2$Ot%e6j7E(-C$Ag7S|e{~vGHnw?F)wucUbW)Q_^KhP3756YSY-V7(Ava zHyAv!o9VS-!}qGca8bEXuMIBK(tB+P=)M+S8`gDnZhiU6*M+uT^P8_6xAQI&Kakj# zy+2s2ba#*S@}Q6{)VjM}Y;?^q)_&NhSh{P&-Q6x#Gc-nptk!mEFe+TL?^v&y5GHfH zFqX@EseCw@Pe#&SET4wT1$MMn+ z!|RuKUd}Llx(jyx3@_mM`A-e|1$H6>H6J>qD`vIW(I_1Y9sctR&SAi7eC6>?{G1AH zp1k@Gg8{kv#^aWWU)JAvw2?Ow+H(EHrFYv%hWSqUT0^OE4fCC=rhLUihd*q;S++#z zh&!*iFSrj&hBnV>Nx1GD+=rPmOD43jUMeBtwMR$a#@cOo?D0aC+=kzoMm!tKHM!Ys zTr(q{4Nh`yz@GNlW3l$jHW=}^Ozk$}nflmcKye-;o;E)AII&5M;kU<#XW|!Do~1S7 zS<_j30)_qJ6H?eOTK8Jk+_t5;5s$smwV*?h!#g@J_&%gA_`aDTI8rTtE^PS)3otyH zVL7nHV+3#2-0bFFC3!ni^>B**maLU}J{Ww)3ntgig%1JbN(X96g}-zvt+4oTG1e(pti6 zb&mdwQD4e)^wpg|I+@}7M<;9e-uSH1-)3{QO$!*EXPB!Qp`|fbGg3=su4crRe$v_c zf)V~7H|Z3AZROq(C!MXoG1l8Y=}b3Q)AnXb&DD(H|IRsc%sCSWn2cVdIcMTvqrQ}L z&h&-))_}&YM;OOTe^&U|&YjnCR``$$ zreE3U7%&`Kf8%ja;jUW8fD!u}wNt+smoze7iw7R3oo64VBtw@zxasuQ+Z9fu$1&e< zWK3T@5)N&CqdU5bgbus!tGE92fkW=RIPh2*4Fz^*F?Y>V)nUiCPt=_!CPS62Pu84@ zr9#EYM{8eFqJREz__njg40kS83OmvZhl-VFtB_GF32wIE%zq@8N9T8#&oeG^4kBtf zaJlVZ?28+mDPJ5$H_AFTaq)j0icF~N1nEu%D`(s-4){QizEt~#Q^A8>DLB4t z(4+rC=fRVy0ALrFY1x|w&|)6x^ZlXaZ)k%L{ejMV&v;kpUgNe(@^{A^y*=AN@37zL zyyXYn)ZB8I-PBY}GRrP|!`i{#eAmb*H!?jv5&dyjF+Ki%er{~k{}(UY+dVK4MS(jU z{!Xo< zdcC3EA-}J80H14|y>qR!hdj@8caLDm;xGTqK2rPX$_);TjK1f6T@9a~!SZEbb@2PH z-QDl%34dUDy+EU5Lw)`fzJ5Cjh#=}kqy;~4eqh(G^zrgLj|g*`*VgZu42YeW3)thWyYuKHd@?JQqZ?QFob=EQe~UOw^1 z45Pc0T;=RY6Tahk=_mYe?bP`c6aM8cm~NjQUyyzJffrsCtOsJY5{RRS;bwpRO&Sg+ zWBpgT=ko75ch>1p%`*J}%Wv~Cmg!TzuuOMfcA3}IGX0{jR~0*gCphk}>vBB*;+0c% zIp(%fzxl_3i|Gh!gLTGHU|d%@|Lfamr9LHlJhZu z+|hLCo-KD>d*R0CzPd+s)lhWe4pt4n`q7kCL($mgmu<09w@gcErS1qWp_RG`Yz_nQ zC9zUJRCO~46H8&Gewgu4Rf+BHmgl;rnk>7Z20CId?X2qS%_igen`QPzReEIbLtT+n zH0dRB=}b0}@zQ=ImWdWnZb|37RJsuJ6Vp{$N{;N3sIttkQa{`Zy!}|IA7T7#|Ee{P z&mC#@bQ~}J?D>`7>pX(l^AH#8{CB;hU2Y@NHALqMrqAaeI;uK@)STfu|J*ms?w?yb z=TvCJH_kcmjWw+rS*TC*&8-=QO>?kAju6qN6~&qZi@)pk(Xr6_M{3kfS1~TX*=(Bc zy!v}nr^mTcv`AClWBbCTR^66>k-&nGo@@<@ipK_Wx39mT%@0o>u&3?o7 zU%M{HCbG`Y`4)4Q4BNz@Rel^<* zH@n%sq54{ZM!-aHn&{3EwOHOcI7j@&QwLmf&P8`!>DNX_4XM)~xCvAa&wc(Ur@gK^ zG9FfZn>#;@@u1Gm8VWhg3;z3{8{!h7yEcD)@)h@dj_perj~D)f5}T{?99G3Q=z(1TZB_wBkm z_`#OVje*l1DxIkP(sT@*#YMPfYVR00{h?CF=63r~>9kjEiib$pOZBP^%2cO3^>?9D zF1>dh&)RHov_D@vXWYxr_jmqTdq)4FVf{blbsVS9UPy$!WG)-`3Xxj5ur>Ga`&(c%4>Tdw(?RT`ER(HQJf53K=H-qX2TDd~+uFE@H!PAWS#qqQw%`cAQrJrAp ze{<&~^UF~#*s0&^>s^6E_Z359!>j#Zdb#lj-@oHn=Znd$wx>Lx#u$kr`&Eapm${Kn zr&tWnImLfOm$@I=~LGx0*-Y9i#^;B<$FeZ`+Y3Qc_ACS<&}*~Z)~Ln z5-h*74C}o)edKJ=PUmxJuaM=`CYMR)qh7|(72>HxGUCV6k)$74_Me?UdiuTz#h{Xd_MEtTVMR~ZTDW{o26(f zbY^k=*$mz8WuT8)n5iCrgh6S8_4rads&dHu#z0I-Y8q`R?n$NxCAbyD)&Wy0jY?A(8 z-}(GAe-wKDSSv>9a4W`>JHoAZb2&8%oDEN1*I=8a{JyT4H8GO{JFi!d{{z(<_2uh3 zT2Pw}6Pz05jy1uF%z)33>^ZrnFNbjIZ4?$l_;&SPlWEVuYAM{uxp z%t>2+xpQ^XaX|5PY-TvmiiMm*19!Id&}}v628W(;>$j%NbaU@HZu=RR`oW%KXvUG& zl!7+dY}`=^)a;_#Y}i+@*|1qL$$V||U@COg*G{Ug2`nW&HN_@WoE;o$7@BN3CThQI z!XeR*j8E8pCFG^bmR_Q&gEr8Ydhn(}(q z)*dajtxp{!&~}r(>J%Ngel!60`@nmo@%1MS^LXyP@3k$7m-k{RKM~2Kqv=E{8qS8J zg;YM3D&(W7WR$0#Q|V~Ik7kzjb#6Z9BcaX5{Nw?+EITg$tagC3yY@xZOA&dh+I9Iq zHlez@>ITBk`Ozd>t#l^og`<8XpW!{}NFtRB`(C^dUUtRnIuAeWy`jSo`G%?qs$I`2ZYWLR^t!h)7BGJ(O&QCgzuby-94S^om?5ApENrir~>5wx|cmcnftf}oT%OL%GJ~Ev**09dGy<`s<{hwFG^r@O}JWw>XBN#%#E|$1}m)P zW0t$bzGfV$O##sjwt+MrsTI$*ADYZ>m9wh9Y{UHKGA-r&7Es(?=eKD`Y8y@1OM9gD zAD!h>-WV$1d)RT>hgv6Dwo84KbngASyVcp4JGTB#bhm~nYs%eAPhoIuAa4%K?L;q9 zh-bpF2-8(QUSK9kMq;^eKFOs{BoU6K!!vZT&Q+6I_FCv-tvAZsk1p1E#?SVzi{<#- zktQ<7@zPIZ2PHbc%tZEb+=87y9r!o$%}v7J1QS{5m=h-toO(Nis;w@9rk_JjRbOUg zX3jeNrRN_&X{q`od4ndFoe9o-oqWUXmPxY&yRwNp>R-HG^`d>*PHvmr8(nf(rKXBC zQ7?^6TH(-!o4!_6#j@$B=HtkwDk)#RpmNwwEMDD-B|;OGBPS1aVlhO?(8hX-Wle=P zK2zf##pD2y{e~^mQ$cCWCEdZdA6I)5WJB$jrnA^AdRUgJy|dU%JuF9Zd(gwWp`-KB zE$6>3^yp159xp|#rzP8Ez9~ZO#f1)LL*3o~pgW)`x>35{A0F-Q=4IwjVccZ#{@&qX z!MfL)u~6z899aFnE*$8;uPci#SCGXWSbb*H8})ZxUhZt29u=>56mR}v2ZnejsUVf- ztsqP5^k}+S@kZ-)%a*{bc!vL<4K{vkr;lm0*#FPe$22OhSe>`e6+`>pxY>AshRS7a zcg678&J8DgD74{(@2*oHaYX;v-2e1>1FKiM&bv|vAKT*Q=b?_y4G(_1BedbWM_S)= zcz^rfxY&B2)!O#y^Q4V$T$B#Aa_@*8ZQ_Wf>S)K((T;T;t>lQdyrTtpfBQSyhVlQI z9c|-Rci4_ro>-cWHr9Fg*WMC3{H{|k!nyKBN7T96vKj4fJii}x(|66-p;@XPd-z!J zj9YX?qka?XT608(h7l$Qyaq*tw3T~%!wnC$Il{mFJ+ZRxe`ZfCAGgEy#0_6tnx5Fx zdCH4#4xRE%>xpGY^zHSvWwj@|PKBPBe1~a8U6rPE{}#+Ion4{vFfTU0fP2?22zNHHgT3&ud)s zbF|%cNrdXe*XqRx?w_`9I<CEFL= zY^{HxN8z&6^eF0{03J3lr$;f}6F}{kZMlK6OiOzM&x2^)3D zCeB}9H5`(rw0+IcA)5jsvrGf2jeRw<%<^IOLlYgcQtg+fV_$O*04`H|$G+(v00N5J zs}9*v=Pj2m3*B<*Pvl#0;&2IjS?7q~-(%XyeQhva+P`z+JBRKRnmFKFPh$pIb|n8O z*Y6Dv^#t9hY63aYVhhZfinLbR9cjor!Z?0ShP(^ikf&glA@73ccJv|dsjKH6@}`uq zamd@a<-93F-W~QeW5}BV!jR`cnhbeAu)NKNyxK3@GUQpNB^~k{#T{eFyHUbk+9A)8 zd`*VDo2;}u(vaskUiu+#;`ckRW~@8H1#i-0r?OqG8pKuZs=bX^vl?)_+|Fsc+?E-{ z-Tv75rJbqpHK$_Ho<<{`$(V>g{C5=TSR?MR?8X zV(RU$Wxrym@=4b>TQRm)r(!ADe@=?-;p$oUMG0g#8+tvlXp{y=LZY6YH(~OJdG;c1P#+*Zgki`j?+Sf?2P4o>ATb z%*Tw+9ckWhl3n_F4NDVK4QVV@v=SFz#n%`9SI$9{dW#{i%l8PQ02kL z?#HyMQ-{`}uWWL5WpUf+Bn%`O`mc4xtBRqw*|c%Mfg27i761EPflXQMIgAGki-ux6 zdvBcb#d8?$%P!xVJ%_RJfMcuAVQBD0}o zgyG08$#ga2C5+8h;O)oydW-S1{ZF5c&mC#{bQ~}B^f}(KYbQ?BLz)C8I=VWRcXaRa zFFZsz(4F`C*7okRqTlNs=o%S97v|r_JN{a*6R$Ji*BkI|(|`-ZzTef?J2KkU=i%75 z%XuBGWsg|)Hz<4Q4P71G?_7m@&_cd@@7~cBU)+E7;HbYMrx*6Rmb+K`^1F_Aq?M;+ zc{aRZ7viC=j_|^=hLm-KvL3Jja?3l$J9Z<94<(hw!P};A&_G)30w&;%Q%KL5=by-P z3TUv@}8xEKkn z&HMW~PCy4I-~94|IZ?|3oJ?0t7$@T$d6M`b=^7<wrO?idUsF%&=Q?W=k>Ln7hH^WUWrwopCY{AGx)bf_r0@JU6;S-e_x^jQ@s2-)bHzw_h`i6EuX~e6XPAm zC^4W;wuF`Nq!PXI+&UFXUqb$54LPw;vFsAy*J31`_|!c7rV@@{gM`jWWEk&I zi4`N^?s+l9gAxl8rWA~KBxyq~p?tv80T0Vo<^8p*ypc7mC|2!7x1pPv|Cp(331nwE zs19Yc<$%Q~&HYtOZ*{$QBsZ2H>>lyu%g`I%KPZWyVEkv*+!lke)1C?Vm@BtIL0iC% zwql^GEF`xOx5YqL`qctx8x)M54zT6680a2YNN$6Ilz>hC7K8KgXSYTERGj62 zk4v{HBNb;kpm^iUVnO{|F%oVfsJC%_3H>XAvTVdCrc0phS&W1yNuRT~rNTj-3zD)c z81MN17-?KW@fpL=y<4L#5T5>fy}+wJX0E(z-P<=}UcYcsnMxMoiCiok_6ym(mnviv z(QG1?$z?OyoHyRFjFJg66cu#;;lF6c=o;z$qR%rg{X?Esf-E-dS4#REK&;$Yjk9oT zJuikIUPPujFIjleukiA`41U8z-qxrZW%O$e$WI81;j*)oW_j9>;Ars5` z$*5IgS$G*g&L(JHiHxB3DmWW1jN1}KU zB*r^bdc~lr0GhvV_1x0Zvu&TlBGjBCl{Y~@UjhBu`^7xd_=F@&W!>c!W zKN^cBa=BQZy;wA!iTZ`uM;VK4&0!=c&X1uq-l3sUF%n)N*riy0ht1V(WWWef(JvYU z`ygP&NO;*8DDB?f@*5duYEiTDTMY8QHuA+itbW^7{=cBOA_N1k5E!lLhfivPM;3o1 zfm}}DR+Ea6a0P*~xQ&*+NIZMI3Z)nc=MdDP^hNC+IUy)RPK?g%5>_&>DMrGr1jf&7 zJG^8%SaS*zI=2zH9g|`te0AM?7*Y;{LBf$T-YwaQlNIbW+h zx7Uc}vyr^-<#NevK9SGiBR><)B+|J^l&9ZQ8Q*Hl=u>F1(Y&_C#jJj&Lmf|0tFG6I zW#a;tPnebE!^N_GhGhk?%>L-Lkul?e@c_eLgO%|P#wUxxn-ZFgPpoAtXBzkA?;1q+ zs)e_X;E{xFpR`1Z0poiF;~isk2gOKuh@duUEde7&kfH8jV`PNUTrm>17$Y`$Edd)w zjye)mlZVA%;}K&cn8e!6h$b@;gppqoG-)gx1(dcrh^GyrJG|h=vTY05ity7Qo*`&H zRA>;@uW}HBN|$6M`w6SYr9sGiuq0(RN${oLG#6ubep_C&5XxK|<%NWN?L3 zjD)AxS2Ss2 zyD~TdDh5LvE~&Du7zvjU)Y;aTvRzioMhtnm1lYD1**20sJKMTefNg^!_qxmHV>`Hq zFi5Z;O$I0J|GHB2OA6c}vr8x|n1y~c90}+f6uH-ZV@~=(28m6OlwZMk$7P!IT}s8F zVB{Hdm!n*#t@_fJR*$WSuGu})^Utf3qlJ%+{mt&#J%>KN`}^O&s&{bZ2)A^@`>!1F z`iJ_sYYhwzR_yh+U+Dk%?t2aNWp>~D6MY}eeKOYX@2xCyW9V+vi4AZ=z1CmhGHa#0 zJQ*)U6R}7km-XUah9f@TDn7#6K%1+}mC}i-sm3Y=mFe<-#EH6kJ-xF1G7NhwnVODbC zpsg7cxi7kEp0;Kgs3U`fKV@*GQw(xnzB<4#=gp6x$gNA)Evw)j*Gy~L zAgc!lSNHkVR=9F?|H!IBFYR(wVQ^q{WYypht#$t)f7K@jM+bZM_~f3S?)h;4;NHDG zd)NiRig-A@A`%Ywgjeuy<>)BQ44OuaGCzLp648{{o`e`Q^=yxulN2N2F@pL)rY{no z{Jx+p8!?3I5;XoQh5!}JQn_4=>$Z@#WKiVZz>_I+kH5k7qCtXP8H~QgVCbb=t86Pq z!pj78w)LfKx7D%{quaU!jlYVK?Gl1n*-mI0g>8c(_fr7R$+l%cN(MBDa98&=w>2ckiecS}_v7M^G=czLf1JwQR(Yf=iNZ1_s}&l#bEn(f;!v!QnpGh8!@1cu>^@)%rB$7}DCqTu(aM;2OWB^RWg`aEah3qv7NcNFn3e6gPPmY^L6Q3> z0M99H%OGhB5-BNgy*x-b{;ByeWL$lf1xX1MxPrTc;z`d|3$7SUolH$5Tb(Fg&1@mgCh5j0iIik zLFX|@gs2Q|7E=s{6wFR87*`F7+{b-+K61ghYLH-823K6gVEEaIs+1HX;W>i3l=P)+ z$JDYB1L}B7kdzcdN(yFYJ6Mhz6uIvLcy1{L%W;DQyD}Kt76XGG5qC{MF%s@4sI#pv z5+6UU3Z)ncPY~3h^hM&jbE;5^k#ICY9ZFv$K7C;oN-+|iA*e&?i$r<$aC|AIYy@>E zeUW&|HB~6ZNH~?C4y7*=H~ydsr5Fi!5Y(acMdCBJR-qIl;a3E8D1DJACrgem#YlL8 zpbn)k5|6G_p%f$G7=k*KzDPX%p(>POB%DD|htd~`k3Cj}QjCPh3F=V#BJts8s!)oN zu$iC^r7se%Jn+W>_llA54T3t9zDPX!uqu>dBpgFfhtd~`4}G-?r5Fhh6V##fMdD+} zSD_Rm;ce-;kebG7dl`dEy7$p2DgPUm;BjK_6FvO}etso)!col=iNVxFa zd3cBg7hnbnNo8<6C`Q7g^I`}tzzhq>8~}B%F8td^}X!q#)rzLD1tYp`wIYyFfHpY#J1~Hv>GU z*ewHNhC#yLmMS@m(c_dbJ2^EyI=DfR`}_;$BUinw6(syAaHVqz6$P`C3-*l$MefJu zB^NYogJiT)1~(!q2E)g0P=a(T6(iv|0tw8Z>`U1$uVo_!)Z><*fm|`9q+oWotIIM6 zT-+qTa7<=VIQprf;J9yKNsjxfXN8>e;ZP)%kL2?4cp;IDCG&nZ z<43$i62+i=E)t1_qw#Dk8P7Q}i%%-*9xK2_I_FcABHc%T$NAkj>)XDIV4PCr_gfM0 zIOw7jK`FZwEAO@bwH+6N8#UvVdJxOTZlik0gZxA>uezq%ArvFwYJz%)&=*C2@YtuGSq z|6#RoijnX$f;yDGg#5M|a$-Q8Z3*h_icx$8vlcE=-Ksj^;==%&%x6=9u?!5s28lG3 z!HtQE!H|O4$yK+i4!C^Y?emeVZdC;db_GtBE}?kAe^sTV80Ei^pe`kSDcetK*@yu( zX$g{&Vn|8B>}*%ZLrq33dg>w#A_Tjr*!>D@MXq1a-FcMWS4Sy8&M@ z60RkvL+MM%AFd%MMxD3>wQt2JzJl2cS8d-8xP1QR`3hHU--3jx3ka+USPX_Pz^k>1 zXRh;2gMv2!TW*Vi?m`k~XEs=*8x*-8tCE_vy#{L>g9Q8IMotY_F%n)pU-Z=(6(iv# zg1U_LMdF6bs!)oNa0x*jN?#;CewCnPeW)1h9e9JF#yc=Tvp7kcEUEj?ry9iQ+F&nl zXsRuYOb7%D)*t^sNSHo9HRT#&);lp?@5=i<&NvklrQy-Bp&l$0bT5G-Y#|l)Gb`+y zi!f`Fh9Aa6##GTPMm7GF1a8HLO*O~vKI~8HC>sR|c#ly=7r3ATRUn4sbyM}yYg%yXs&;PnqyJFA~FT=lcsI8fg38zD5xnx$Voym zYEJ=_Gge{kq0O0w)vP?exS>Tcn7M~QbX{}O7l|i4RfSTFgs&0Qq4Y)Kc`sC<6eD4h zpbn)k5;;YMVgflUW(oK1)cx*h2CscsyL!VbyuQA{To)EF`N4g=`n@6cMk_4ey)IwH zHR(T{mUmCfySuUikC@9R6f1()zFE( zVkBHcP>0f&5>Ki^DMrGz1a&BV(e6(u35tSQ%n~;3)cxnKQ1q&O{p{Laf8QCF>_Y~> zNm#EaZ31ieUpMy7QGaxeKaA~j&_@|;vwe)b!-E63*)@k;bzthl7fM$))1y6g?*%&&JUsKu0H$MN2x2&U9dkC6W~R1cK&$ z=RaUizXEjG_pI;yvo+$SV)EaiFT#lDTaNy-Qej^z@|qGwm*QVTFy8Sc_)&~Tu7I)R zH}FHFUMh}g+eR6ITn7V9+V*E*vFup)e(g|3~C3tPEK{ zG~^XIk7x;M_%@u>m_uD+IY))H1i5NMSP_bXCbdDyNA=`(8cv7{^)AIO>zXaY z|1VUEW4%~0{CMAJ9rsv}8Cts5W+#};`~zRZYGG&}x2%`{ka#MZ45yR1w4ci6!jw#lXXCQsiL7~YpLR_R`d#C1RBHWdc`p#pqoM z7_{DQEw#OD&^L@69Zyu7^bHs-7Oyu(iw}rV`5`!o7@VI>fTVV-`j27Cb33|V?U`FM zJlNkm*5A`Nh~1*IM)#tba4hO)^SN+5 z#i7P6Vlbw_$;Ty>11A3bRrkV$%SWz>ZZbB?Hxp3ozi8-E^1y*2?r;w;*K6NZNQSfV zYzl?jL?IR{WHEw_Wix5diznjAa5z2QfkQY$M+?6NtlaWAW5-tD{s(XiDMMy}TV`&p zz%0KPcRN`q0Bd5#Ts)8;9_-DpT;mPr_w|PT+QhsP5#g5qgOQaHH#tWV;X=wwr1Ehl zX{Ko=o-kI^DbLGi)4Ak$#{i{pv?>a!BTf4VVt;CsXyZRMf*N$%M-E2@l?P0gr3alX zSIr|!C&Cqnw=x{dF0m)LbO7oKt=uB$`m*YYX>csaHXfLmdXmOAI;ngtR3sy#~WCwHxbpg zmt$a?#(j^DMhkwdkmn{ip3S9Gc`xGSqlpO1t0?Y0JZBf}u$P_%d-JhZlc9d43KcLd zIP))h<=3;1u!CiAFX7Ank|0yK^yWFysG{ew(%v&sy`bG z7ZPFWARLM3@;-i4*pjhL^E?mJ2ga9@qg7V$>c^Lb5@dWaO7ss$@YRhk3nlkyWO=;F zviLP8%Xds~Ff0i3{a$vgu*Ua>OlXm8!cY6jY^)GVXX4>lBpLSDy~g6{M4UDoAMao@ zX4vR@C!5e@56=>%1*ksc0huwSjbq%MNh~#Oz;NfSQ@Ma)$9-DHFi& zpESt0>Mz90d$E*{qsDYJonS|m4M(xPPvQD2no35Kxcx~-SqEl5TJ;xVDCuk80khky zDn`Ns1n5+-_(Ae5Bxr~c|AR(QBcEa z@@X?A#NdCJpviLIHd8+r1e{3ZA0x{##K>|S0j2-rhT8q2Tg~L-(O8G9<_y^mcbc@-Y6^IL_8f&Gx{b|oQuWN`6$bjbT-R)^7jE*h)Oi!OHp*u2FA@XqN1@K^Wgi0~tOe)uyzwdFuMmP&C4 z6OQGgkz^s6$tB~sVaVmUMa|`DHeM#4U@^?*nypwxi0R{x5;R{9a6@muQnZ&}ww832 zTMx9H7sC%PBGa4~u9NLo+TabpHePQ069nB_PSh`j9Zqf}FeiT&Ljn&ts>kkP&9$|3 zHDU-qbe|(=(jjcbc*VM4@zh_y*~gK`F~sQQ#}Ock#^Ec^aiuy(1m|wqWVjG!Rgz8@ zl6YcCM0_tAjeD^O-DSMv|D`zy4Gf$}z>w(1jK$z&95Q4YhYV*?qsC$|A|^wnamdQd z)H?mHh*e$rTq{TMe1ll{uF4&=e=yF~jARx|6e6i)E}Y0^{R|e=UMinpmCeaJ|8fQP zJ2*)(BFO9_D_?NPzTkb_HdEx3$VgHCi;ZGYe!kmBPbCx~TEC-Lb}la4M^WScm5$;K z1eC7HNZOn4tN9JdNAh7m%pAz|b1s`qd+CJ7EEp^Brd~Re&6|h$l5o)s?$F2N9`(GSZMj}H;@g3HR+W6vd!l@yn z-#lTQYB$N)u{f;sj?xRR{4Wx~*nRaIMYIqekhJ1&#=ytG+*TAzr84=ntOzeRvsYb z@_EsG8ZN5nG~-jwpJ>91;*&K_=}cF61@O4w`M7p6?z}0+QhD|Yx`B-(y=>BtM?62B z;i4$(N4cOykzfFsI9e42um0p}p#*g@?g$poGJ>z}1(bOro4`?XAtt(lZJgpV$GrSTN(-&=xQCse`;&klKo9bIeqW_4~j=|4hk z4So9RT$Ou^BYvwvbf>3}0<+JU)eDJK?Ls1hfs>cTi*-B~$+9|+6;KNf<8LyNj`+-@ z&dZJHAjsPYb}4?|7JhA|iC(1&6MKKn#s{d_^oON+ks>s^7hq3>qIWc&9O_&Z`*w zDQMDpDUmi}0vl=|u`$FIp-os>e9u#a}{vyh9RG3~m&_4NE*mwIz}m z3xm>DNB*q!M*he@piM1oVeknuD;FK(Yy$GVt-7F~#W~fe;gOzd8%rn9oaZ<{74yTK z<44kI^nfF2jCy6*8)oCDqXuCD;dvOMVjMq2;$Ao&Mq!$ZR^JP|g9sr) zI`C|5IdR16;Br0^2<9@?h)bY;ZaB7qlGB{PxL^eA9c+^pS%pfqMa}#8s z5KLK2Hnr9hjdMyF#ZU^*==nU0`6!D^K6w8DEsX?M|5DNsGIk+P5Yz^r#bf49#Zc0K z9A<6NTfni}kPsq!1x@6{I}zn)45IX$L3F>bK~5;3uJ#MVy{p%Z_IUln1KA$8inL}G zF8Bp@>9Il*S$RIAd@@x?acG!CK`5SP?<1G>!cAqpK)@!`or$-ckPIP3f%t>HWW%FFwT6YxR5+gY@Ub7y6*8%~=OeAO@5RIUsPCsZTBZ8S zLPhl-MZm3r+lDU|uTB<)5Xn2dMUHG+!L6Y~%oE0K8LZ4Gi{(}iuAbnyy^#P;-`yZr zRpac>emN zwZVuDZLOn#8n2 zLQMT$;qcRgTE;!YsG+59g}uozs!8q!NB<^+Xfk+@42})thXw~-o6Qt>{4*BGWipXO z#>+%dGAS?|ggKZ?;%fw%Q5!`eqfECD)b0`+r<-lXtm#H5DNle|1LflJ!m%M_x^Z~R z?$C4bm>F}6KQfLhPZGdw<7hK7IMBOxa6g-s;)xh%o~cMIALTeBnn=-LlU#765_uG* zVjLzT14D&$pEv0S$~lY0%b05jk*&j9d{=9@HFQYUVJ(5R>6p81QqV$52;ufclPfr< zpEa?e;Cj|tVX>?YMN|pI5Zt=&SXFdYhiT{2_|t<~2I&=__E(SbcDQp|tol0-Xy_(J1c@ zMzL@VdznIxF%~naY%aox!TV4~0GZruGATPN{`OFlj)2HO?T1FKBt??;(4u5Ug!v<& z<pA%53#u3aO?5WLwj4;UvZ^n5Z3*dYtg*_H}f_^N{tDs3!n__G< zWOSPsOj^3aU~{#2%o@#wl6rX2FqhuhDvk{yGIw|@%WTY9Jjbfd;&*K{IE?^q-`jY` z#4*x{*~Qm4Y*oF?ZGIA`Br-_8L}2IH79-(%B(TqQ@3^&@2_uFuMboriEJnijjR|`W zr_C%FDcZ>mRXJD;7L>q#b`PuIb~3?)W5i(MMrbx!N__x`oA?Q?9yW;j4z6~_xN2~C zHFrl|ZjHZk&FJXR%07R!m)l=w+T$H8lr1N7&E^>hYO7f5y{L8d`=i6ypt7MI7#Zsi zOzL7uth*xVa3q$+5+KF70E4WLQb8`9EU;u7@6doQ+_a!A1XUA_#vSb<>9C_U0u;s3 z(XzQNk5(0jr!&i2tvIEpoy3+;pPwK!vm~|6S0>2}9ZjJ!>3cDpRHP%gnaSrd5W@C8 zuFFqU4h=6g`m9MSH~?rV5!Je|BdX_&NJ(09EhV9H2sdrlk*Gj|1yf6j*m~=G*0hR0 z)IcGH&2_|RYZntxs3xvxnhln9pCY{J7s+u#6DcJ0UNNtkcxbk&f%~aKQi=c^z8iLm zCL+xb-*FOTzXEgxK#Uh|+-w6sGE5Q;YV#IH>Rrs0Z6Ks)r9=VgK4yd}-8L<^fs|w{ z8i-Fpa~Z4P4yg2`@l*WQ_SCp83Pb^J6qwnW#ptC0Wv+Q?-EEMBCILsg@-(!YOakvm zR>fPa{-3&gwiQBeBt5XoAI+`m&5h|_el(TrTZ!i})9Ni1e`=*Wga8r#N#lsdyrI2D zF2zAyE}LgJp1}4qRq%KhGt4Cw!+yLF;q*v{$wGr@jwE0w&c2rRxEP%5hzuF>Kgw{H z7t5vgazX`{3Yti@2Z-YhqIkMNEO6abwY>G@g(#Cq@qaQB^P(6TC2_05OS;jV@5?wq zZCt1+m4fDNoY~WUrE^~m5aWp3rYvmTu}#bv0hke;=FDKzX}<#WO~=e%(pp;0ii#}s z)J;Ie?cZBXM`!J2qf`Dq0VQej@M)9DVsCG)H^m7R2EE}}5{(+X=lHpppJVUMjwYA! zvUx03o!x^`pbl?S$WzUp7bjX853ZU-J&f6prnqB2CBUikefaVZs+-MD=P&Mj}j^yhbE&Hx*ZsQ*6 zRTw$M@JE)z0Gz|hLOK$SGDdQj5j+-b1Spp1G;a+$ozSmLwLc}G?CuP>eJqI#EGcNt z5=R*JD=gjPSkk>)dl*t%c+eOs{+z(MK~;>#p;4o z+{pbW)7OO+ldb~#$j2MUk(pi&*)-NR?9-KkMtg@n)Z4~t+CedlsUjYKl72Lr#gZ!R z`8hlpp&XRN^wupbDvpElMR3r(J0BwLF#Rk!5!8m5*MeKeNBQDexjjhS>SwrB0Jj=_ zmYP=$6qgwnB?Z>DEJjmS!t8A;aIkDpH056bY>UoWdrshB*&r#jGPtI!7>(`OImrdD zpcoXnzwy9)&x&mCD8`~)8izg(dZD3s8x9HhSva;ny%IbH zo6O8NoGomXD+w6JcWs;?n>p#W6Nmy+ANT*sbQTu`=>%JATu^$^WJJfPXb(9vb0LP? zZxA$}{sFC$mg8K@;#fVFN?=Eu%OyzSndoo~Wuypd2W-$Q!j1H9 zbp>&opq3MoGMp$NvD+LWEiPJ4hMF?`lt*O7J9M_E7<6wZu;FRCMgf{qrHuwry3-(V z6X*^3`|%t>-Cki@U$N7PL0prWK{>+3qzzN$a5fiBBnw$Dn&VnLo56ulD$H7uq8nyL z$Oj-mKXAu&EhfXkyy2(21!Sbz++rdcbcLO=JV@XQrx*fPK;iz-WZYu992HicG&04f z2q;*SqpE3@6zDJn3LI%4H5)u=p$HmFB+~J0I#Y-j;wWTi3n=C#+)<%WQxFBshcq2v zIH2Ow08v`kUE0Bn2*8Yj=FG_bn*%C7Cz@aFg?R{H>9kXmhOCSE8VK1vCwg@?^Ps2(x7xE_K?cYKEC;Y^4c(*}x zzqLVXpZ<)kH2oa+9exVGtPx(1z`Hum)iX)7`XW-qOfj~~)q${p>>dJ4Tio(w5h!WU zh6~{bD5c512!vXY3ls9b*9uHmoW3B}rQt&k;&FrEf^M3vRD*izFi$J^!>H%=4vpd; z*m-(N;p!$G&U!5Q*t_aZIO~}&dTCR|q|wgp7Y~bymCEx3h|6tI+eD6m$ml@s1tV8x zsoci085P9*q9gYISnF$3*=TvCBd#z}Dxh3%pI&FI7Ct%T4}1o{cGj5l`CK}UNk9HS z3aJ#gcDX1{R#ND*VagFt1RC)UWpNyoRi(B?4#~qxJ=EW73JmQQtOjHQz3&>1yuED;pl^`#Ay0$#hbC^bgVdP(PUd7Q8%@gW<_7>aFEaf)SA476CvMhsLKo& z__v)l?$U=}4bO%Vo*m-SAK5t60@TlV@`;$e&}A9$roQ~nA8B%2snTGQ*nH-JAp;5mcDBrg;2 z7-$+}#6&KfOS3FUW>FMoS%+V?C>}a=$sv^Vf>Q}N$Z`GwT7_T7P(lpnry2f|-VJIM zc5yGXaPP1e-BTp3!cQwIpJ(_>=M%tr%`VQJ+*JEJ-Z5pqY>hJ!i{`yTf^}FH=cqVj zz)IXpB6a+u=3`l2dt|$0_z=p4CX`@{(GujE>V=gidXXX5I|Z%bTMbn>;afqIhT22_ zxXeLZVGtapPMiK3wjD!*`})J|Hv+F@@l+lM)x6OX!7>hyRbGJsGn3C2;yKTg(ulM! zg&H2dL4cu;J9%t19lLllI>oCT9f`5kR49$1r?MOstwz|)wVIA>HE(dGxs8C5G_ijS z9*xn34CjsMSQHJxOcXD&=~OK2qe>m)yem;i<5AsgZHqz$=RY?21kaEy7Oxf%LP-7o z#PF6vt>MkZ;}gRM(t8bQ=_5#PQ7EKIUN=#Mu4{I=x7Vqaw=-9LeFrdph;uxp}glhh)IK3@Zu$A?KPT< zN5VYW5{~+XFzPi?c`W5^XI#MAms|1)HI;S&LCx9JLKjI3r6XMc5kR6B8XYah+EE%q zk3=1n^0qDc7FrtWrF^NA+T{e>P!f0auQyS6f6C|e zbfKpIeZ%CXsftm2q4ly*I%?@6fW#afbLQP@D%w~$Diud%8<*k>EsZpp%2t}n4+yB2 z@H~ncyl`FcM@I*H*5E43IsQwfa$(O${VW=ZMbWY6q7yG!nJgB2s7)m~h(mkA@Y9%Y zgiLeo30>>zSAgy(U?yOpV)9c$G^uXfXk)zp#<3E#+c}G%5^F zBGzPxR0Dp%*r_~h5WL6Os2!KjPO;XargQ$t$m-$2F?XuKBYG^M(rksZ1!}nvk8#hD zh~g+luL8>?Uo|fDbjQsE&1)R}1p5`BX91$d-F00%7+MW25up|WT7OK1w}TS%ypf<( z&q2_1bSy?&Q$mwE(FN(3e{8MsJOUVd&wO=Kbyc#>Pso)-0p2DF+>NPXB-~8EbmpFr zYae~1L-Q25@eW;~Dn`OBM&DkPYaex^LL0c%sB>RzF{mqngIV{$T$||AIgJkFZiifx zhM=SHJNXF?e`*ljsrecLrnDUJn?q!_m$|-PwwktgTnzfp%JzR=EJrZCKz3N1Z9y0Mc2NA!}*aeXGakf zMQD)Sre8EIisdq0CcvW%%YvinZ`u-ozhhd-gZF_Xuxcj6WzaN|O}}hPBgaG7U87#M z&xZ!PdV>S};70@S4@{$hkGs`=*I;&UWYSd__WiED-jUJ3>SPK7COE85Kzccrf0cEh z1MAU-l@1$teD{WiR(CG~3B#qj|M>2E4fJJp-}@7NAI*I-*6;7FEF8D2axVlBbD39Ex2~Mu zE-%EwaX*ser5GLn^JVMJyNBsWhO2O%-YI0V9{QxQ1nWdPnDL9ikAdy+jw@uss3-+d zV5(v}LPK5J`0YBsA&%OGY233dhU&NE2ZRK@VY zX#$vHO!dY6R}YT*U90A{suE+mN)pdkW`y(bwR zd(fs0r7`r#%TXz9+kwVHOT)2c5C z!;@*m(>pZc=W2MWwLjpGuC!lP_U_}m9xcDXLPJ0Jfy1^(BQ~C2?;BlH=p7wk-(AC6 zA9JSt?(i~%8A1(V&#E6w*{OR)BW^RosE$23tlm~&?-AeIVM%KUBfEO(^H$dPRNHlp z`l?aC&mUSdIIurn7yA1L2V7PXX4=-phAY(K7?|)TSKxsJ zyvm^o&A(Va8OLE3{<6%b!O&C8bD=^p-8o-7XmN;PL@1GHHQ$DCuwcJ3;sgw-=d^ZVfvLOb1wl+2JH!k(H1je$YDe@nlr*=Zok6F zeU1@a^tO}@nuwu>4N-8maf(6men;~6t(mm4?6i&Y!;Z$W2W~;W<%c1%fVM%#;u++C z+lLQT#^M=oT4=`7)8wEhobhF4EIloSnE6{HyRGuY~^fGDz=g0Y&&LjnL~X-{xbC+AKEe(}wuL zV0_m?_ffLJ>ygJ=?k3KtScV3kk?S~u6jx%o&{cM@+~%;%V)tFm?_BAw;8xxt;3*Ev zNj+S0oU!rl#(J1;{adwqL%Ww%Q-l|nQs`OYD=eRm6TgFr;D*+|FOG zV)TL2Pqb#HsgVJor8Etikl;5#&z@n#?z-c9rgP5XF>L&LosXzky_)1Q#g`O0$g3l8uSf${uDO|k@_f&p~HJfJ!g z6AYbq-Q*(+VZz`5KOOLSm@pVXFPR5ak`N3ve&pmY7s7pNk3Ao?xu@o>&NGYVQewf1O|+%mxD!>n67@1hc^bemdZDVW##B2GH-!1FGT) z1{ohc`Q$>FFgU=^0DK-M3bcgu&QYF*hhoFjfd_ z<*3Q07s7|&^I449J*3V63u#vk=Tw_HST>iu}!-n5hzj0rZl2Kq;QV&{dC`EbhF3HJrf# zelg&46;CsiU;zF0JfJXPFbHkkd@jtA zv0wnbcpgy6STKN2Odhl#CIkog27>vR5DcJO=K&QHfqg8`y7OnpWIsYpFtcN_pDFG0b6~O`sg@UJ#{}p0GxX+$vY#p6 z98fBseum;pb8uxpLQgQWluz>1&(ND2%6>))b3nnB{R~v`g4v~RKf?I|z|78-{S5DO zKta!bw&rzo^Y$W3)FUU4mZVM~fZndv14*u~dDgsoJypA$bn-xD%$fn{vbgS!^ zSUk^5m#A#`p2s8A$#4p@#~igP#PDz?0SCTLQ@6ELJW3(-$m49IQk2GKYiUptLrzH? zjmleX7-%gOltIc@SV=0+5KxLHZ{Fy%vwB>ba zkV9Oo{zp)w@~K*PubsptMI^^fTB5)Uf;d0N#4n6nH@QIPAc(Oiwxp-)O1@v?+cHwx@1NK2|8L8ry0Q8_J6>;fbCFb3RyB8n;pZV)qG+~)zbNSMcQWQ0s_Hleh%e|qcS{(MGQ^9Q&_cGyq+T>nQ0Oz$^A~zQDckdy8b>7bn z;&RGvp$j;4$mOJwf!#?mfxlgkTF78t70F_~7C3HkSWC~C%s*%s3avn`H;#o9aXZv@ zkKYQ+)+R48NHBc_Dk0!iih2gLt=UO%# zkMm|Mo^#{O5m{W|6?}AyG`uWw2r?p=8tL8a#l=O zG)6NKpQaT-j~%mJp1?(0w2<+&Hp$T&M-iha(Ph3xz;!S~JNJ8SCOeFXMGvyq7+JlB z+-{Ug0a0`asDd_^mF85u(UnyJ`*HJTdc$rc#pir1<0tWzQe({we&r76ym=KQ%mvZQW=u8rO1qk@R^NHjiJ|XcYH(;aGx~=JC|HK=R?Cm_7sj-SC;#EW>@c#cdNC6n09dpt%Cm8aQ!4#RCCicik9F z{GhdJzaqPUANi+f12-(4i~`&o1UII7XE7)%p-HW=EUz4I1t^_h5Hq)~)D|`)tNp@o z@9H(9J%j!G^_VwZ>;5=5^}?A{D#qK0*w7}^wDvf*6LH+>`pG=bvS<~C7!HKLYT7?z z=ekHDT3QJ^o#|^v#1^Bir9lTUl*+k|i}MVk$-}69gR4z&9`9(UqS%^X zks6I-EgnxMGYRyA@W&?K6ZY6@QD~^k^9gF+*cu;OT{uqkUJMzP;c!;uU}7;C@iJ(r zko*P4rIZ$#U>1W@%kitNg@^K2l^g@4dcQx z=fx5n6Y*cT5YOWa-owpG82u|Y^@bii=EWAEW^rgZsfAk{YR&^&fmr(^VMOQ-aS8oUIzbC~*2s3R zp|0FV0KY4y_l32CeYkBhn|Jv)=h2CMR!no+ln!IBkxmzQmO?IW3z)+f;?Y!|#Tkb) zIlOnPbqYD`-%n7RE?ZAYMm!>+g_=>SXr0@7IvT4*fSOj&WUT%>IH@>@Um8Rc*WzQ& zfonLGOr)du1m@Tvj4vh}k@``*CdV1Kne6a-B`4IvLYsYo03BK95VFx*`GQe?f;tf{ z>f}X3S^85XwFrb-zX=nfe~F-pNFZ4{>NX>JG6CcoeRPg|p|5w;b>)oD8@TwoW6kC# zvRH6OlVQ9K7P7e0#^(vg2CPKO!UV}j37Emy1JacjiB`jo@F8u7wxXxu7l{;J4IA=) z%y=#Tn~7+VXeD*wgI5Jjq|TBJ6%i9s@kd&+X|HiCF{E}J0Yz(EyVov6v;_<2bBRJe z&wkazdOjIXa}btF(P~p!EViN;qhQNpxabZ-Sv|j5G+rHA3L!#scy)NQXspz4=}f~~ zI@<}L$vE2YaR!5VQQ+t!&a*pt7M^%E_prPu#4*L-{s15EktkjS+!2M z@r9p2x_cBK;)tAWSBAp{{L{%vo3Ju!{G5PO7LGs|YgHr`9*7I)wG}nP?`QVOy8y`HnOT+ynzXuhiso zIEsxk6k)tm5m!YFbKL%@U7tnai zv+cO`-i44-rU19j9a@1|mFzH=9cIlp%#p3Yt(Dkgj_b#roEz6ROp+Y?22V<5@bnk) zBfO^(iNz8;UduIOG+#)>a6*<&$2oLhB~}tH+N6T&Y1pFiYWeE$7N0P@b^x|$v}%jO zN4+U%(iZVuP z8Jcb+!UF~=ZeqqLsz@Qi7YSJYcxR4J{IqXrz+GE`~b?WU`x z{oxN-X^$Xi+FM6^`+fP1_JSRu?|TUbm|WKPA~7yM^j=3KnZaj7I^svuoZd5R7KIKO zewm;;BsX@}ZAYykMmQ0aLtRnzv;s4oz)-U06y_$KfP>%Sgrj|>K{Pr3TpeuvlYWkO z_jqk5jq_Xlw=!DD%TNO6ls*q8Ge+p)7NJq6BcOr5$Ch%9N2&|QDT7BL1LyM&XAja@ z3`TJ&R7fSjxW**27@XQX6dyF48wlWVd81io!{eM|?!g$|KJdaRzrZ#ml8W%`ryh&J zcN4E8#0s$-3prg-39p1efMCNW*p?E}LR{F9%dd?{aldwyL%10M9f{I^nCM$t4qClo zM_8{wt4XW3lZ&5=jbibkU@=}kffz+Uk$_^)IN1yICjy^sb}H=lIn}4A61)930bdQ={2RL*IN9WjMb;qwx?LNb9iODq>bM}l`PIL1$V zdNFOhqlY0-NU4|e3D|$SE7#Uh;G)t{Qy-3k)ko{_)8Gv;^&tFB8a($>znFBH#{`xvFMIPl4)IYXQI5J5cXm`6PXT&(VQ@EN`@Xuib{2fIO77R zC8XrfRWuN(BPETrb`WBMEM4yiDS%4jG=XzMcQcxga#z8F5)q!5;HW*!dmmwTc&Q9x zOQt!_WL;i%*oxmb33bgfKQ0_4XWxbkp*xh4#A0~VqCuDtt%4@4Nu{{UL2NRJCMT!n z-3g_46HJzKH$R$5g~N$-f@e_D5uT}y<(cy$YDp}wIMpi)7bEfu1hqqmX8VAAueWf- zY%dHW;(5^!ml1hO_%(b8$)T;h(*(6>eh1+9z=tJ>%E1Ki+C6=wYv?}CkCvO6h6V@w zqEj~|5&U-~kZ>x&+tpr*CtZ2lHOFhN9xuIn9ydUv%$A~ra?+$$OE|ii&}rl$I^*Nfw!Yl%pl1KN$>aHJ0i2PBC0a0 zGm^z{vZK!a?{oH<_t|G}2kt2%)!eWuufqhf300xA}n)t|8KDw;8ay*Zi=0 zj`b;%LGRNpgFOaRsU6IAeA)K0MS58dm)SGgXe-Cqf0iS{ToCb$1v_fjupp0X7(Pbz zX4x&l44Ba-{B*Px%S6X7q0{l4iDXkO3vCpg#ZXDk(#~mn_+=}^@T-<1CuZ7K%tg=6 zWj<4399qISLu!Gb%N$iHH-o5JC!AcHOI+3$W-5b1cHabXbnN-RCzmyu(H#m`GnD`l zm*Z*zD5A6m#|q)7Lr@u}^OhFL8z3G3TP9a9Cv8xdsn7SUKE3Z-j!a;cooHo!V|{fc zJh!xcZX+gG(e4OmTmWIMtH@?SlgQZ2m)MJO=D~QT6$LQ|9YT>9G%ax_98wk5UI(%B zvXKp(@M#!lqxAwXOZR~h)2;~KnnV{l0)!bPx>e^{?j@6U~uhW{cM(G`sYk`tqxNdSf9@Jjwf ztzTz3a?TSM938_TwYY-p0crJ%4<907_vz6wvIm$8W@x}v_QS?dZ|(Gp4<9jx77n98 zb_UBZ|ERIk(?R`$oe~sx{`jsahBT%Dga0w`a~hM4$MEY`g5DjLBiH7DGZlNUUFOjx zqF8Wr#nP^XS#L)Hz&YpFs)V<#vMEI;i6jk?rW0^FHu6jkE&yE`AHmWG9bLnrAZ(7z zgYOyk;4T-*&*UsdYg_C7B7&HPyyUGmJC#Np@M5IdB~C3y(4|JR%ud7mGfO|z`esr1+zuHq?eIgx@7*Y)i=yE5 zh)lQ?|6YS1-UK}3x+n(PG6)$4_mSkRGIpc#b<5GaQwuv~>R-(-hNKc|JL7xdinm#Z zH#wNwml|=5-jHo=r%|ewD)mZ3w#)cYkV@Mb^qvocbonWmi>$N+&-&_Iv=5neZiCQ6XUBXt(i|1Ok*(HK{jI=P4DYYxSMsTZ9q0wa5lhCEz zXF&nb>?jPArgi9^d`cGpOgA$`hKYc4x|!TryuoTUe3RukQJ-A7GuqM7!)N05@cQ$j zO5&)a9_RwQjtFZ(yULTjpw1*wmX4a+BHg)SA7V_9;G0c?iDGnd90_@CtPreoG@a<; z_{joBjF7kUOVpu@<9HgGzQ;KB|JmEGxu^Y-Q`^~<_|j!megtP&>8xP6e=c)+BQ|2K zUh0JSTWPksLAlv(@(vo+Lzf62<{V*zIY?}fr;=1{vq+Yx?~NBCb(Weh7RgZCIj>{y zR+DIEJI8Lf|3Q;N|D%>8=fT0*oz>HUUt+b1E!u3B+341>y5lHZZZi8uHKO#^nn4|n z4!d1LqK=<5v8tQqK8`pPEYqOjG1Y#`VC}JgcDH*dP?c3^bO8sIaVsL46INT1&l&9S z^CYx!cBwVR16R>__fR-7U)E5SaRnBkorzn#%<1A!z1|2qt)N5nz$UYf9<>UR0)LT& zZL;4+7lqHM&48KJzGV0VZPkiG*G56Gw4sum8Cb$tnRvCBnZd*0OuQK! zi()x6zo>dF*&y(Sr-q}7HrC})fO%-W+AW6x(JVS~z1e03WH#-K1!egm2`iv?(?#)P zZZk&kyVv>2S;eBbsdi#RKS{WG$6gdaS*t$nF5{zmj^Xny$9|#PN6(|U;a++|d1PGLEO(o2!i=#YZt(Q9O3-xN4e+9+ zix-PPW-QVl3Y$7!LxP^&yZb&d;xC+$)X3sM&vQn7o(wLeK8DXK<81UYk(qlD^%?Z7 zKK)maP?yr=jI`&4zqIblyRO@T?|l6T9n(7Qzg@wQ->{yxN%QS?^-%rOq!Nbbci{onM>w*IACi>n+EjH|@2=Q4uD( zm>`uZ0SjkbM^i7YQwvWH9lR8AthY_OF`?o&!E5BQ<*-}Y**R*{E>;W&=co|d3$CH_ z8zwb`ALC+iHRMjP_fD%}?{;s$>}^=Ic5-739VmfePL#q#HJt=$>|p7+66zn3Tj!vpZZt!b)r}g@1k1P?sfEoj#4l+D z1y#)KDIZbJWvC?GE-!+WHkHn*_eEo+jq@TH{VZKd9O!gP;4*0)L6_@)+x|ZIj{iN| zl&hDY32XskHVk8)UsWrdHX3>>h;c+%j=EgxmrGI}tIG+8ro4xQ+0cz1=K__Lo6x9( zgKEc`)8!6VPQzkoIAo7ACEaeWa?+0t4bXc{mYII60p5GIYBKmUlH6LzrK_LssJR2|w?3`8StV_KM#x;c!2-<1=507cfy2x3C$*_@Rwo=4z|aj@uQvx!{SBKjar2 z9pFnQAw6!+HF`Kawsg<}Mf#2{g+h+e!{L$XTL#IP5OPl6LW?`R+w1iomSeh})1PE@ z&pb(Lt@KS%y%RSoaks=B4mNm|DjM-N{=iXdMdfZuYLiuJDHhaHN!rJzsyQ!^owPL0 zZs6?H*A~T0+r>YBj_Nj40;j$9etU<`_*GVV6U<9P_JxFOt<~{(jgGa2$7c3BA&2gA zL&go#1v!`A)8LZcf-Qm-+ePC+V>zooSx}wi0Z|<2q0UI-ifzmyDy7}ai=5Xhyp^(7 z-8OTMkA)#`6~X|mce57OaY^37gGH&{=A0N~Ctasev@!^d3cQ(wW1;(9D*#$A4TMP} z+~UyxjaDcCm@*q8Wp?1yJJ^!W(7lqaR*uFaR{Gh$RG0^_YNdU}FFqf}v1e@V0yyV;J5M!l?m&<0i z#W=@&F(}ojIS=pJZS?06YqTasq0whvNirJU1wdQdZX9XAM~Hd*+ALk|aFJ>##kn^BJ6Tm-n$W>%STB@D38n`qPv8 zI<)E=;b!bZu*~4AR=bQj!UnZEm|xK$aj?@s+CeCbA?o=QL1~o?RBj4n36dp;evT;q_35s+)14wO?*^Lv6!kvZ%T-%?z)#M(e%N8Vz1E zO*3rFje=2os}be}qS^0y)m^Q%qL8qwL^x}O?N(IbvOdHOG;h}MVpQg==C`XtM;QN3 z!Xm)b$|8^x^JZRFQkRJbtiEHcdOSN?1S3>{#(}YyFnXA^@B->2-LZJkMt73r%zs?< z+Mnp!o@38duMV2}1Ao6eA3^KX4=qRk>CZjPI)x>ZQ6TUej4;PGNUmp3uedFIC1g%u zuU;ufWi+d04E);knK~O-o;FIcB-GUR<+edY+ZG_X1fmJrY4bnO`t^28qqf+D9xebBmS)VOw54zH?%| z^TasZUfw*BTFWb>u4Ov0g+s|lfg=ELu5g^1?ML>z*(PH(B4<96&<+vM05e-Vs%OZujL8wU1dAl#M@bm8$leOxskTO5$5Me(k*TAj1;a$ z9H@*l>T82rJReQg&ZqZzYYNu#Nz0vwljBQXl5biLE@=*)3iNRI^7XZN+%y$1L`-e& zOS$~lKMvN`*G@#i>e-dkOBb)7*;rYPHy;~s2MXNuKP^g@jp7cOJ*RU}OR{*IwgCVDR;bb89Q#>9;33uQq)44Zm5i=o@m7s^D#{@;_# z)`0BREjMH9CL=mgn`}T9A`2?9Y)?^G)Xv76x_wIg-Y%!s6%o#$aJ|WGRJE+b4UvJd ztU%oy5FqnYEnNwBOda0BD7xIN;3uuwZq>^8`o=s^rfRyk zA%1|li#Bj1B_513Hyz3eVlpQu=1gP3gz+ikH1xefun}K&e7DKtoP11OYf|fd*`=0M zSFWsuT@TM3#?2Tzs%{O93|Gqh7oYQO?)V$!DlX=b5ZgclLPFp4a{Ej)A1dlvgF_nM zGE{qpJRhb?DlqzsW6JkUuDbRx_2@k{AO;-4EjuW5 zyLeHG+89ohuz~AfG{L2eA!*YD;OJT`dxdu>*vD(5WaaE#ZLoSST0mJ1jehQ+1HXoy zv#hkt@FiZ>Ye>!?`DNNyzYy}ZB-S!N;pc&^a3@+nu@$o_M#1Je&H zzbNIaje4(%&YpvwsT4-a^DZt{LL8vuloopqZsvH1Id$UEFG9v3=*N-401YeW!)QG z7`$BG373L9{6H&ZQRp&)bM%3uYf@yDC<>ohaqwm<{ooxYcCe?mm9#y|&GhqLBg*Sy z-Hp|qlPl~$ZK&+hh*Wu^-sbv>E9@@O7kU2QAs7Y^Me6vs?xb#<1k7OhGRepgGH0;N z!9{~b@YLcf#>H;7i{-(vH*jj=JbJ%q%~vcJ8nORXt4aSqNQ}eOM#LP)a#MXLI?pQHT-4 z@CPRDqjsuTEYl4Jr?@dVohi3i3$7>gqwzwnVwpcb_U?5_{MB1#Z7v-zlp9zl+N-!5 z`AE?TetZ3NY$}qa@wS41*_Cc9XmuK0jE>68nJw(-YBU^<@WLAY)46*xaC)nF?4yY+ zfDuM}#(}=@EMuf**`tUofQ^|?->BRe`yjW zj1aPA{>QPc-RVX#jyS575YuBnR~r*5eI>k7HPqr5a;`Q`RNom&%8O&CjxRp+P$fCi zW}fQ3$a3^vmgbl`Pw^Pba?>eaVMKdlw6bSPE_ht^vAPnkx%F|}WE((a-%d?o__|R= zI|K}IW5P#v;wC*_Iwhe3ejN$-GXBJFPn_9#SjYla*fjgAorQcs6v#lmT*x%Dl6<)j zuD2Y6Thbg;<*p5xa#QYs5#{VTY(eO*@35`jPA#3oiWC+B?oOh5od?S;T++nm->1Ri zP(tY1YYUQ&avKS^>2B4S-P2qczT8;}n3BKG@a0LOu+mH1f<@d)^2Oc%py&8Xnq#WC zxrXg4H$wTE5uNzC$>o?n=(khR%GSne;Je>uqr!7q1qmlE0k_fZM%2OERmi4_d8o}% zQNj=)#l3@MbW|(`T^^tWOQ{{*EX#|*M$iVz*c5h-pw}2$-*xVHTaKwgTSlHA-B-_( zWVD9+$HWS}#iS5uzTh??k`v9CI0h|VF>`;{?V{)2rbyyM&$^F<>B4*NiOHrKEnUH8 zbZM{|ADfdcTPNcO+xv~Jc~@F6Q|fBm=r{>e*H0~&EhqC{?-mnr@76@)GJFcZ=muAk zST~qDno#6hZit?5L|D+|-*==<I-0{Yq`HQR)cqu zP@kXAhR2);cAmQXXKXf!p-Vj7m_QcwS|ehh5|g;z;(JhORk)BP>_zuCd%Yv3>oEp& zER4A^q2Gv?)!lx<{42&h zG1OQLd##mX1Gle{%FWvK z0wc=tlVh(n%F7o&)^y|?Wwo~Z0*G51%Z{?}i5G7Y%fvCf?`XX7;kur3YWLvJRD)I@^9Ra_{GZu{cnjty~%qpAxl=5Ps>y zr;`tZr}7IpC64bOdROsbD!%2SeU#bkXzUkKaO(Zem8nj3yagKB(fBe;I9yyL#B`1_ zK5q#eS|%M(!G-5fh58WXid@0EV2KjT6GV+8AC$ui59^TmU;p;P540 zzgLjVHhJ-WW~khh@p>c5iGyPId|TV=oAGjZnLFMhig{G$WPqs*0q8p&CbSw3x48H3 z;B}$v+Hyk#_(p&^FZiqIY?ze`g4bzYxOAKZUK^Q$z;%P@;6n#b+=CVbPuSPq>%8nr zD0kLg6kis8mtQM2_4J&r3XirztY_K*k9~Qs)u;xg7J}wE#lX>qN>XLpJdl^rDIBu* zcLphi&pd#1&B4uHHWv+%)k&8BxxRyVhAf@^pz6Ar8XH!tXq)2TmcwI6)bMCMV`^_w2?E&H_hS;{_F7pX6< z^wNTJ0?`S1`Vm%eSB8%WEN@!)$CR6nn20&dyT0#P_4{{Pj=YB`6Gyb&?!Miw1m92=YnYpc0INZ^e%(q7 zNB8ZxftDQe%m>oFg-$W0z$eHp2O2bahI8d7VV08Y7&Q$sZXC*n4j2^%}@21?Sc*`5wt z7&|?Hb9O^#r;XX7m}y(~43pyU3X|e2YaiPi%guaom5crtrsyK+3>4|?E?36 zigg^fqeh4Yb4|~TbS=(6F;s1Y0nZ;Oy3x>I=`Yui&@o*_S!_8Bn{tRnemQjQp}$g& zmwGvfMZmR3#(;2QxE?|pI^qbKQYi7e9RJY!Yv;&{PNNmbQ)m z$9J(dA6Dc8r(Ub!55C#q#UYV~VoW}pJPKhY(mkP6a6+dKUF-GmJE{FtaC5NIg|B$f zNY@!Fa~HP&Mw-6Gfl135_34$g%>r0y+`h(G4c|awY^H9pG-{Na^1acB_S)@xz>#Gx zUbegIn`bhChXM>y>o~Fqu>|YH9k#RW2=mD*PJeip5TY$c-D%kLSFuR95_{eU%+E)6 z>taBgzRCF59-?qIT84y=#FXSLLpnUX)p88}G0icxA(%ESH>H2-XKjT2a(2p?v^mCn zS|5CLF;|w+A2hKM?rIM0a0^dlk!~AjYo#(qou+W*rue-(HHEX+ z@BbOUklNKGN73GEWa>A0o^PM_j?67oFmbfh<)zosW_*%s=*bw^jkv<(7?r{JdqQ3iMK)RM!n89-!-~IqQ@vnWtwC3 zdl*y>?FL6nIjS~*b7E?P{c0=eK#6Ha?HA4T_k6I`4VAd^+duT) zost~X#FmS?DYNx$`vt>~8+DG8n8YIKZ0c|W&o+)C?^Yu=JSb)`_T?5r@+HUhzT5wAW&PL|27uCq24^-_D6dQ((WUhA>t^8R$d;b%xuC9jxw=RSwbh{$LO1I;BxznIw@%zcf zqus{se5a(SkL`f7!6xDoAK2YMGBVOyC{t{sjT=4dwa%0?*@=a+q@Ko$G2<*{KgIrJ zp-dmF^}&SOe;WyPOs4u^JHBjX`{*h3Q+Ns3a`yn1hw>#^4}a}fwu4BW$68%Nqc)>j zqgJg|!v+x^h)LND=mL0<@@q{oq0N<~JEX-i)U~EF?7iI>YV))>cDm_%ht;qDE)sZ6 zQ^bs7pp^6$gjdPz(6O~5bY7Xzc@>mmwMG-&3ENBQjD^bl@{c6>Dk#kp`YRp$y(A-@ zalz#fHs!c&-*RxKwZBphWuhE%oK|2dw1pKirFb8S9h&SH9Q6T?9tgBRp4xQ@8=fB| zv0*%Ud7z!}hdsybmSg7q;N*_jUbK&*1C(oZ!lhuLS#4Ll)rk16jaIXTd*HB*y=EIV zsdmC*$RJj7=$-I<+}Q~mTjcT)<1X`%A5qP9Oczs{T}e)xsHgm^Af}rR8dcm`n|x zOH}c&hiXQTF$7QX?;=4p&RtGQ(;ic?YP$DB{?qB@vL0#G$Za`Yr25tIiBv) zgij+}?bYyGhJ`Rg{cl|8jL&yw+G;L@74N-7aXspkDIpN^%6Jsr;Rmqkn6f2XbWTuO$31iv$@nk=aGnggTks&|GY|(l&hh#qt=Ic!OmPFlWca^v}SU4`1z=vQnG}kr~#YXz%L{F=>i(_7^Qp-NzRPL zZq)t+*Bi)j{>U*%^otKSrX)Y(yV31an}{0wuaj%R>QOp(c$)2~mRjY6$IV7yN;>-u z#x3^gLi9+LL`fG0#Gz<*LjoJNu$v5Jy<5UNMJ>Xc0YNEDNoRau_cqwgqokXiDM~uy zMyE=cUZ9e0c9xs;Iq^c$&ho;Qbh9(napmpCw6DZEi2Y*981TWT>?s|@aw4hQjKHr{ z^qZRar@YAKbT4%Bt1DaE#3tE_H*K^x@v2|rh73)+#jO?2n zyZ1g=q`Qioz~dLl0AqGTMgY!0yNlxBwaXAL;!=`7W(V(g=_#=$873b-n35bgW;N=S zlM(Zw6v2M+;r5vj^w3B-Ddo@zaxAp{<>uDz1;poYH%A%UJ`#ub+Y{>;jKfA|Jdk>JP2?YrA>-u3Q}{m9$Q>vr zJ^j29xGP!x=F^oa##GxO?U0U)D7Ts+5mhne3^9$a#Jp%M@vgDXX(_5V+nk|(vzI(UHpYnd_3%G`hyeB0&aDe;_( z+m|0t9R0@8@iza|Urgs=^oa2D4zuSCTyS=2``kvnwZ~SjQ!6z}VHlw&Z1Du9%=sUy z)^595p)*I=%IS%n5Gmikk+2DNO;kbfTpAiKB6j$m?1%~i*90e2M&TDsD7h2dm%MgL zy#107Ur9+0Y(K4r%8B5W_ym`H_}WYen&6ZZgdd~?QztmQgIR8f?lGdh-n49!fS!JH zCB)2O^Xy89~A5WH>%Z^>T!fnr`$F^43v;>>S zo)~O;AOn%p;8!`y%s2UGf(@drjI2{|(-)zAuCUn&%cW+w+Gtd&ts0LLq8c%;Xm|o? z8WS4+1@K}`@T-EwaXdo@5G!Ql90#aUX17(ikQy2z8tH{r!yY$}%npmIA)~zaBCBEV zWhBOJ>L|C!W=7O^qWvB}X%X-JCZf43OGbrrH zm6XLK(ZepWpnEAvPq+KLwVIoqE~A{?(Anv%yC`NF_0FvK@>KMhQQ!Mpm$MRI29gi2 zN=c?J1Imf~l=w1`e7J5V1T6!~3Buo{1XDZ6B@ih${rF8rl%qU1$0Joc>y&C;Vp7-3 z#IIt$MvUZ0eq;Tp=EZ@a~M+dkN!g9MRWn1cJfji}0*9 zrzS~GfD#IdqEF(*SQ|M_^slyU!t?uh#tc0|NFVHrdfYitmM zxTSOAB3Z7{f5eOa<195VE|MX;mHxjP!+{ci4B;P!pW%bG%R!d%mWyHiS)*2~>K7lr zoKjDn#F!i`H$-<#*%p}9dfOv-zoK@Wa{Rw@9Pg~|}=-U2q+PcBrFZ|_wm-=~wzHfkOR zk#Zx7^G1{tba{Ja?c8N&R{gb4quDNX0~|VIc}HXj7L9-%Vxy#mBW9_v{e>VTVb1fn~opH?Vg5S6F%bN^JbtFGlc7`7nL_ zC>K5HuZ>!F0Q!Yol~DCPk%&+-CX?965NtKLJn@Vl*Sqy*&>?z64U?a^9ma7z#wRP! z_-h0bx8=!jk*?s=8LdzrEKdfG0Gx-WutGU#Kj%W~^-V~5$NGW(L>ZLWZp?o1;bkew zf%`f7q2;6@USR|$j!te9;%=9im6J#6N|PtvTt3u^Aj~75piQJWT%bj~`3X9m4t70K zcLD^^2-rpOPFX za?NeZ3Bl`Af~jkVTr?>+L~l2uoV)Vwwohx7cD2$DOQ`r2#g&j)jd~@9QC=kC*0@}zFac67u_^ERn zr{c9Gozd*~(y5(er_ON~rt16bpdK_4DP`Spv! zTtPy&Nh7XJ(LFSbK+koc>?1GRRJr4Pzh$dotJlU8Mun|KJLFLo z(KC78(V;h$aHZ*Y_kyE$KA(hc<6DX%uw~jTP)h9BL#daFK>jtP{6a(ST}NV_;+H3k ztXIv*ddXgi9hIY_B@M5gt)nQdFmpliVw6<-dIPu+o0eMY(7K9WwUF4DSdkL58Yf`w zR)e<^`0(MKp&lV^A_?)KSKL4{I-p*_gRI$^%BDr!5Wh1ulhcK=q@Ko$vEwXtNM@g~ zP^MY~U+coXk;KAFl_i11Ob<*Jfn6**w z<0L-#^$XsgAer9EkES;&Cj_520Cq!wl_%kc zVrBc>G9f2dccR#4fNG~vF3AU3RUTr?xcg8PA%&XYr9vyl4IJaffWiMN_*@Y2Bqlop z%*llrz~GU`*PV-W0masYO`U<$>BYkuX;S4vyAJN)Pt;Y3I}J!a{Ci4r(57yvT-4z^ zMy*qhe(~W4DfQG-&N4*GO(}n9L??b_@-#Y{(WSH=KS(bp`vPp?8s%C*9$xMzB}UkdC3`D&IHLCEi4LiHO_5i!+VM4JYfF<*!}bj7X~gpgHY*>3AoyDkDlSeprs#i=z&8g4Oa;G!|4@I zw{j1kBe#(nEpG+g zn3d5VT8?{w{Idv_id^UDdu+6K9epQNC*8QxWkroU<(PP}Eqq^h*^PzR4pxJp)288> z=M5C>o4{rZ=9d=5ao}uV&~UifIs9kjQyf0)sR1)13O|26?Z1UTkwb~knaPK@rX&X* zlC&Z#Cu@ijL$F_bc>7EU7~NS;2;OZ37~LJb(^@@qn&{cyKSq;*l*FQE@Z9P!`^;0z zr{YyiaD4K{C3mS6peE)jzSE7{#1@HarFw}ep@sWGpS*gDH7uA^zZbTil(4=`N;iJlJfuMU$foe#>?PD(EUV&XOt zPNI$5ooYKIly)6&ka$9G@fv`eE0e0>QSnazi&S$}@2p!F0FG1*l`=cHnK2epZbp=0 z(pLq&&xrb@3rvZRsN}<^Q|$Fl5=v- zE;lMo;!4*Tq%xsu2iSh|2ry>)E@Mf96AK+Ng-+9dgJiVTS^z6~?-dVZ`b}fim$B>u z7}4j918pHjIem^hi2k>&DENf*4F$Ef>%^VKox%AbUB_&gU#*ZjKfAv|If3) z@Kmtk?q17{Y8bQ|UD||*5kvwD>$Pr(?+Lt92F*4%Y!9U9LZ;|S@*M@~Cd4xh?jh6Y zilX?k7(QkB;l5>IM`?eh|0ok>VYbin8#|9wv}6jMlH5yTPUY{^p-z2f}D$FGe1UwnH-R~w%T?zMl)_i z_d|q@S{U-_O^NsCOD-sblTUM40QKUC=00v zg^i${g|0W)TNlbiUFaFkM2WX%^5L0$Fpksu!%#VC&A&_$>=z%NH7kPUq`^dBWw2j- zc+N}+3az4?u)NX;a>A9c|JN#FyEPs{gzY-Jv}z@YD$R1UENe*ut5izu7RI#akNiFp zj-Y7Yt4Pp=`Mx3>2IX+Lx@8j}jq134^_zK7nb{4F`d@AJ&$FGdKOs`(a2e|tA6}4> zOlxP$i9|$T2=9Ub&6Bk1s)!y2;5jEN| z(d#=^o`yTN6m@K>b8pzA~_SW5lRhCX~CEnN(@ zN-j{^`&KXcFW5jW23>1}U}+~M-qOj3+xRfkoG?&M>aN5P>=z&2KP!Ueq=p|z5$qQq zJ~R`8MzC^Hi;o*Yp6u7<3wwE2uc@>3s08?jC^hN<0j%Oq+$FL@P;PX2Bv?nMitG&# z+%JO5&XA)PYMdO|UjRD2#$f4_UvhNrt_s4|DPEwIT!}SfMk_w_9KA1-!`jV$A@#4O zB-3UI%Srh~UCOX%WCW@EBQ%mJ` zsqFI&bF>jcf0xr|Z3@y*(DB+?n{99?QbOq$bjl%Wc^pj;y?_uQRn$UV;(2rar)3Qe zLHq}(ql>sAuw@A$P==Rd>q0EY65{Zs*2?0@y&sq?GAqjo>UEFt9z5kH^{Vj(Wj|?` z+wF#n8e*eD)aPd=-rY{3qGKe)Sy^tKVtw}TQ`Sxd@*}jc-or&JNe58k6K?Y1ulO*1%Kl9n zS~)?u$_P$0rYO(S&XiH0QBA|3U0&``BTKgN`LDaNx^r@6Z8@|@{MI%&Zm9%8t6k+_ z9<;i(dLwE^m7tB{r&-4dp1B+}NZRIBklB^_id_^wbGrc}g!dYLzeWQUg^rfiU@84= zhTdyfyA*}5bz8v5U5U?}$%pszVR|P}H&RaOuf)4i^5KIsAz+R+R89~+ZUmU49jt>* zw1s=Mu-q9L0>b*F1O39}eGOaj)(=akYh1nBM4`qJ1@KiPh7#wQwPn=&Y<@h6HsPm zzLXsL6GiyE)w1_|X@&p99jxB*g?%BhF9m;dNwW4rrXJHPSp&SN%y`{56J z*ztY+uP;6JQBOMa+Yf*2+G_jZPrPLHH^bklo{66*lGF+N`6KvyF_v)tf|V`Di%NVM zApxzuxGF*2Q?8U1+$Sgz4!%|4bwa&T>EJ@Ti~2`iR0PDtd571;tq8NeSq_7WU9iDX zN=KE&)0`z$s|bv4`wgpkrd2e?^e4C~@iCTs_*P0XZI3lnPI!DfMX+Cd_|8lS+TkcC zW%#}kccoSuAm^3Ag#$QD64LHIKx$hj6`DVMuob0uMoU1CerM4g6ZAdj%3DlSlH>QXBT zqJSvh7}YaB8y;eN8nC17Z1#5K0>E|gB2>!k;5K;{0?hz5OuD|Htpz3zK^z~3PdDbn z=U5Jo4+k${3J){B>AD1Z5h9t_I(_z;ZMb=EW7~C6O}#4#ORaVd4-MFT)bMtN&tEKf zO57wWCXlYqgaqIlNjP!-terULgnc})!b1`taM%ZNjGYrym#0G0++?6VkIObEEM+x3 z%KBY{?JqM`&k3rs3XQUU&&wM6fXpn?hiWc*nBf6?C$Kp^tVNEyopJBYNsb3in3q3p zGhXtI(bA+lZ_Oy80lfE4%?Daep3E;Kqr?r_k`FKBgRwu*kWIOOdy!FVHqP%NsM;#yMu}Ge?P>=vs5}WEoMOZ(u3I9<9w7_W+Pl#5Yi)P<-iu8(d24&`8ke3De>dnKdaq1L4r&a`h3u8t&G+<+ zY4BA>J@PS{fj-yi_yiHB4&8#*t=Q47VSrX4hHGdaomqA92on z_M6JS#Y8)J2xIEe$FVkvcEauBX#k_~oi%sd%gdeBo%o5tx#i_xXM275q#kVXcq>}o z++5c}Bhq7pJ92M60%rIt0ec^R8@pcx977>;hCA`yodk=DcF-h>8L^AY5k8|r{BCy| z0T*ld6T%mj;?6jy-Yr&>qqYbwik%$5iw}L$*=Z~`xEOWi(tk-s1vVChVB5`=NekGw7_wyPUlTieeD*A0D9NGaFAET*tKuu#1BR zBI>y*x*=1#_foo430E)|ZdeVTKIU9ZA6GCFCWYQ@Cg}bLP0%(e*l+lL!ezh5{5>|zrE4n)h4F=4Q{%+&4x?QfN7pDn)Gy}$r|#{$wgjK1P_-l z8<)OfuM}Pr1E-%SJbuy!N3mQmaqE4>YSQ~Q3ALDJ;&yg@HL?e_J6q$8zhFd9r@ebN zm{ZZ2m5ln7^Mla?(|ebd=Al#09~MQY+jei*4$_L|n(89ATeYyTbk0*3Q-j;kI_~c^kD#quxcuRpqfp&3+|?eRi-18_7xMm^;aRLT%(HwGmZn zt)O1P60Fv&w@Te8Dq;KEAz}dc4^CNUk`LeF!}QGn%{)|2denD}z?`ewZ-jTZed0EG;^r$A+vr&8 z^6TuNq1fD5Uthf_4LZ@Oa~s@@+EJU3k@apRZnw+$xa@>+rG{}Z_YM-8xWMc_5=N=N zuUR0=HCT!l6T|(^Qj7Wm8B#~%M8|N3TD}-N3uLJ&zxQDiYwt6oZ6m*^w14D!nqBuqA3@cKQ;jsx_OkblC+);L@$Q#5Rh*CLt(6Re zoA_(QQUF)DJQrz&)u|45QIi#e&$OxiCI-$Cns8Gx1yyGbx+Jg(gs9 zO)z<#!RWLg!pUn2)V?+DZYwd? zPej})DripBs*TnirtrayU_rhU6GZs-+_4Hb26iaWZvni&%$=)8K|tk0lVys((a|#Ra`K;OOw;Bgi?n|0A%m=&8 zh&bbf*X06GttA-h{~Qt~Fl^ztXNhUDr?wI`_pR<&jXs$HG-qj@1esk8h-*mH= z+NY^!{q1-)-Z-_sb}pk9!LrCezhb%AGbZ$MBFAwXs);OE{g28f6@o`cq$lYNV1dvjCoCndIr>?a>yZ+Vy-4^(JNrJz7d;w4{ z&fTR^)~V~Eq2+p1n3VU62HjVT%L0H^W+BqezvN}ESQ``ot-XRU={ibsnzjsc&(ZsG zn&W3DE~c>(A)4Q2+&zkG`qda$XEjijnF9BQ3F^@a8sZ+TY~ogPg3NZ87cx#Jo|ioLb*FvzCu(I+K9tFL|VD*+e=WW#C3afr4I7F!=zhCJP?3sp2+=?Qh zdU;B9ka}yNT*yPo`6Gzjeo>RZHR_yNV1L=g&6`bSA-c-2U8;3($kuAMf>K9M9IN#X zK7EipSM++#A}CnAl4PWb%F6t3xRSX5De0>W*WRhli>jGMa10nFIWr9d+5SYUBzzAs zklAS%6~egI;Ljgn!)3pz$MyE^XS(VaRh$HdalKjV5F!eXCG|?RiT8szYH^6u zJR>BE^u}-6Jo6X+@HJ~|cPL!VGXg|5Z!z578>Qc#6IBO8!O?cN80zrNuFRejSDRn~ zA_XNm^OyD#o}>5HG{;Q6M%_&m6!~z3>TXUrP~zP!`SA9%2nTi+X0?HG!umr-Fn*5E zU0>VYI{x_e&ieNH@8ZaIC0u%Zd~#=%I91!*Xma%2T{JXbLi&(&MrGHLxOCX1t2?#5 zy|KL8dc0jOVWZIPc3S05T(6+W#oGp_Th#ASRB1Ca;oZ%!>B=95Q)(SpI3IZj6$7N; ze6aO7Q8*Wx+29eFxXeDQKxhI{>IP8_qTCRD$=M#y^sS|h)nHr8^OC>iTXGxDrC?*lCbp=> zn-P4!Gnd8qTnhdXI8EX!mL9(U1B=jrgsl3hz|QyU$BHY&rN)$>9!LY`cw$l6@sD zS>`y0Z&dp$nZ9kAPE7TU>QBD9{^V=i^Q0CL1R}rFk*1jlg~8THm5uJ89AE{4QwVOE z2&IQ|6Kz7ZUc#pr=FH|B)qoM>z2LKN@;jztxyYnS@Y)a<7kMO@YEpIRnix2ey5D#V ze$2W}u{F^$CwN*`$?WqVDhQ)M&?CZWDhSMgEHh$rniD5>_wcAuL6h37EQR$-uvEiI z-%=}R21`Ud=+N+S6gR57`#o@mgM@nDQrYK&V`4TO!9D#dCgxw!2Yw+NB_^An^wUZQ zS5+(Fvc1MxUS02S7NInj4S{&>$m{y88%r&$x|d=?#qgr8%44l? zPvr~;Srq|SM^;{!-jmb1{9I05w$4ze<@L@J(F1K!EA)LvK&_r)bSEx4tY0wq6Swt8 zxCh&E&tTf%2o2CH)56SnY(vFvG>GMSUY_hFz{;7E%i~~GD1)1X7;n{EQL`NfTtgCt zuUc!PAr9)C!t6rCz-Z?ef;mzK&yUQ$*B~jaBM)`eH9s!Pi2>48Uz7@P@-~t7wm(s) zQI1*KTf(QC7sGCKEmECqr6%O-_8h*RGczy0RyF~Hg6?C=J zBqmXWBT~C~H3mrgt?*&Bavvp&;l{&5x-`OPiG9gb+#rEx@$!{02Q9YJgx%fVLXJsimw-%h?aMe8b zsK+1c@|xqK6*Sx%rg1VB$nYPX(=QxX=e2l0N!-`tcv=!O3Grs!k;u3i%!< z-6MLx2TOgzD_k^=;kUU=q1TKIqVPH0sPR^-T`gB}U0R7*7_q8owVFJl?*v$tBYI;1 zpWEO=r^>y}FNhnR%Gl7*?{jW?yS|v3Y1$JLroH!@1nE$d)1GeN^(S(Cz;c{FqM1;? zX|dbw)7Vc$uHTcbv+nXlD~$+%`nc0&=WUna`kBk@a6@E=7(Gq9ispv+q?IyzR1VU- zoWR*kr<+{83k+d>nuLokzaX9;nU%&MspV%4xzAIAoR#JRN^F3X*m0AKocT+PS&=~^ z-=WL*_vRb3K^Vqs+e?={Cj7&RA**|2ae&e1oYA8vRnB_leth>dAYYH^MNNn)%@Vp3 zzU3&OrTi5rV*k8jKm4#u-}cIxSguol@7zYZKcdST976v$M;*By$?dW`r@hOvuQCGY zzHW4kj~SQEKZH~?2EsHmXjE)MZ|CDe*^ zQBrE-zD<@14 zt)#g*OAkaq=eyJCrkxG_d6GwXF5~fv7fNM`%R#qN!fymRKrCxf_!uw^ z_XF@p<}Af>;VfJL(sk59QqIv}%3q^+(-CiJ zdB00%3SB(nX9URL1GCU=CP6z<0T}@;?R2#A{wg4qPT45k2WR239gIsS0($C)XQ5k7 zwx}|g76G-o-RbtcK-goMop^$C18qMTE$f(J3wc#Bb@AMf6@RrvgraJ_Ok_x&+(gZ| z6NaefpM)^P2X-GJ`KcZ^GK|%Q^4x%@m?0YHIh2Zip)An@8ZX*hEZxS$%P{-|A8e=} zWMg8v=mVcLYS~um7a#u3sHfUij%*(?FIQ=B1EFXM)*eZu2iehcjnw(y$dy2IVs#ptz-$;%qaro^QBqnHpX;DKqbFA3*E_GbCU=t z=htSTi?@AF!?m0dK=&P|`_(Ci?`LqAf%}22==3<`?N}uzJONSuI~_0URBo*F4S9H^ zhYnFW!hvY18&oP$SP8p$pXWJ#g;R-EMFBCH{eXm{xNAw~Lq%y|aMapSrO?Z9Rd>je z8W=6^m_6UT&R0^cRf0pF_qaT#^uH&K*D4Vpx0&cRlKuccBS12<(Ix#)cLfQ*`@Nha z-;BBY3}SEZY12TRew^6BSoh|L|-ME0J-0`j~={$H76^5wRiqN3BY8#-R48E|LG ziK>P(K7t^$t6V(r(pmPN0-*(-O~UcYZO|_qkuhRWw2dR$%bE+vL*HgV)K+*lDNgF! zboH!1QHSSPjww!s&#Wzl=QhVvl2HMl5FgwHSCg*9&@}?8d4)60ZoJpxZ6ZdkuAmWe zHAOpW$`c2HzTy(@+OSBE>mj;}I=%%gt}Yr7g9G(SP}9Y(2xy%|gh@pm+BPvoAfwwd zNahAdRv*eei5y-aOUw{n>fy^w%;5g?gywA0df%bpeWQ^KxExU%A7*x#g%?TfRxm6_c5?Od}$rEcZ-*nT?XT5-qNZ! z2oNbKnc5r9B)vhau+#P4It$(DUAsULFoO+dp$od+`?Ow)0J`@$-83gMI0nX(EiTkD zeF*~#1EGZm&$K%KRA%ixb?J^9nl%*IF^qVSULwFqnXn=Xs9p}RI>S$So4Xj<;R#1Y zBe5N!{R005&$jhg1DLKgHygo;4T(Jxv|q6KSi(jx!{%lpFfn<~s4o0sF8XH@_7*)X z59oMvv5^&<;3)d7Bsu#5`j`C)zVEdhbN05O8Py+?P<`?TUu?<1V_6o$!`$*6HVqhrZmXvc)%M0a=6=&r_hCo!tB?m zrwE|yU15XtAyd|j&49NN<8z)_VL>T?>(!Mt)#(?KQ!;g`JDtS3(^Thl1E-_+&td`d zi%c^K-3yE#bpP46Js2~d##^3QePJ7=`xj-QWH0iPjZ~ncoyu;<2$!8%-3T^$3T8_k zb|G!tA4NFlZu3MttkoKwMx)khD;iQec3VaRrtMw|KFV>QkBa5O{fogP@V_=L>hXJT zM)oKY4_gyW90EsDO7i=u{fVCJIi~ctmGQCC2$0cCbf?aZjhBA&om2!k~;X9P0lDP}7$Rb75*jD=@0& zn0l2k7b5eNK~c&#&LXUn$&!iTwhSV7;ekDTYm-oO$MHqNi(JXN}WcTW|=7g zHv-1cP5Xk&z*v8!1fxu|KEN>wN_A*Lqd+i+CNv7_b8}jsnTpz`j-sHBO01(H=$UAd zj}cMY!!d=>C@Afl4^>*Vh$txS;Rr9Ls0raaj3Z`V2m3P70e^Kph*&i7S-Wv!i|J0$ zUqw!Z|H)BGnHhBVs=3G2ln;I)ONSA0V|XiBYw{irYtro(JD8$Bvws@a0>&wm5LRe`j6%SfZN&f6>f1>7|W6JD%YJ4^{0xI!-L!Ls{ zP3UlOV+7Fsi_=YWakI4^j{CzSAr~FwpKsjMj1S!fipfi&O1pmk9CYb_J)}%XC@?A1Fzek^(ts zGwB?L&ViS+<9C1a*MLdhfEjk5buo`LQ@JNIY|&1g!$g_pF#_m5mo`u0{cChjIbl2} z&lB4`k2uDNKfA)vckc$wkp@6Bf5CBnF&*|}ob)OY^M>tX<`9-Q-{+6$xtfTey~Ajk zT&F%)!y3nO(;u8@s#kJ<;GNu*j@}$$Rm%l9kw{HZ<#OY|VS@sm{uZ@7p zjp!blbd+W81;_3Hj;{yX$Air?JZeq4&1J{@smZY|x?RYV*t1c5nH?Yviz<9G?LD&; zMH)d7P=yiQuOGH3;=F5mPhDcd_ZGoSq zyFMBziGcF zZa{Y2tozK$R%o`-_Eci8xpfq*o?SV;uQI4md-F3+ms!v0|jc^>c_ms3a&4XaqH%}79LGM6!kq)%aw98p6@I)-f0=cRod%(bQY!(xPEFJ<~|XceXxp>~T{s3$gL@lX_e!)d|tq zE_FL~-Zq51sAJc}ty`Pm%VH2ukGVN8$u4<0THN3%5tt(tOSv&jD zBbJF9{!&=2*E#`@J=@q>M6DK2?pl$0B{6DinEdGNxiIN8SCXXnzlGszU(9uttf=v+Z&YTaVwX_s~7K% zNyD3{vd5))IFb?}l2VeGBp(h#Miq`RNXqcY-1kFCxh_f~I9;@Vxzaa&)XpI3*a`A% zB`deRcFfI22>r{`hMIA1)}tuVK>8{po#JcR#%maCZ0~HwyKEtvCGOCQZ0o6 z&*ut728?M#W_of4Hv1iJQxkCTSz2Q@(9n z*TZ4Zphk?mgfU-(TDj8NX)zZh;`7q=TA&vfu#ZEh0n-(ek_R(2&uHpcI!u7Tv{t`P zH}zWe{=wz7G($JN*un&QtFgH>QY0<}w%;1(heg4z(8!T)QjSN@RQv4WIxbsUM_vvY zD3VREe<@Q(k7)M?apwsu@ud^5nIs{R(JKd5zkz1%!yx=V5)vsAQUdkGGu3UsGNtvx z^uYj;nU`#+C)kIJ)@PIm<{e(JJt6KaRLT<1Z#heEp<-Q{IGWYsAm%<4GlCB9I0yvI zgS#js+$%QlVY9`BC|56rOwHeEveOjJGE*=cPJo6_J)F%@O;j)w7I{ObAH2(i#p!<1 zHd*I*{fXLojuVeKL_1IHTs5@PX4bwlfql2P(j$-iUBrb133W;zx)_l51xW`|!s3E= z09L?a(0RROu(V22?&Vx@IJePu`4IEjxGt%zsx*?Q)G0E_Z-YKDX;OAK1JE-ED6#JPa zhBIgcH_zG5wTW}uDltNELdv-sPx^KIj8qz7mlxu=9C6=ChK9613}|XKUjXtGXc0lu zB93hP$pV0NJaFKFlAPlKH=6b*Sl{lF`sHk!?COBlCgTpl5@mD5wxFw?0v*bRm#FzPuZ=IBBe@NBHdo5Gau<=L~VS%l)cZT@|xWEGSKZ$h{bc9m}Q_(w$DV*VM}F52qppCZyUE~ zOJ&)qdO~%WEtS#GCq8d=_$VyMd}U&^o#0z+py^32RRV1T$Em!?Ly6dLH&KBChTrT^SHUQCPz|< z^3mOs>L32@0mq}QcBxfvwn~KIDYeT1u?GmtQ`NtT6>|Ov7HozN*ZW|C+Q2=NEr1m| zZsUPuomG#E^##?*IM5ExD7nF3K%LlD84ud%es7~$wpHUUH=YjJRv82hks!^otr{au z3|y%sgo3COuvwO{m5r@9^u?Ja%qS6I0x-)G<^bb5vGN!>(&^n`8kbp?Fh`lFk4z1P zfN&C^oQ-*A3ER1}LLZiKtkt2@;WtP3uE|hV-%sjzFJ4m2rS=4tyO;k_b8U%xs60~28I_?y9i$B7>o%WQoN|qMcH=GY*usIecUxs*cuwMSOR+Sg@R6WQuYZ3NE0Nuv+NTn z>{hsW4!;(*$=A#;YNY!TukkE1{}^OkO-*ispg0Lq&g5pY^BuI z{o3w{bu5I5W|5tPgAEd}gG+WYd3vbxjnfSix|~%eGw1j`v2|&qzxruvMceNpde#}UyQ$l%I{g@6UB1H%Z!hQ#mWOHtOb@Aj> z{r7u`_d8W2R1@`;npb4-+@Ctry*kM>fiDFkL;N3b-p8`i%1n$NX}lqed19$SjlI7-U>2W)6cuE(y{sGshTd zqHfHfhJu7X-p*b=TddlrCpA(0~+FL`zK~f3aEH&H}7j#rZ zqo?yEFtgNfH_Sv04TOPph#Kz3ny4Wgdc&b;pLK!#YE}uMrXM-Z9$y^l^N!*)`@0}} z(!(NeKormMSOjNT#uykzK*J5a9J36JOINxpE@C}?6e09qF?#quaAKE4EVYyIQSFuD z=;$~QN(Dku(8SMpt?g`&rUuQUnjT*{v!%dSXL$XT*3xn#H7A;}mbk+(0_d(VX3$N# z!|+nBuAGf`m63(j@*~unyo^iPMq;iRwb35u<`(kCNDY9MeGRqjqL=Tmjs~2XN;QU&)Fqv6qea+SX7*%YouZ=sYEeISu z3=3(z+2uLQ!DBzjah3lNxd{-tCAg2u7BwTc{ctCwK$93I(oCRpnj|9yV$p74;})Ls zLBjHG)YSVeUhi3!_c6%0raBl1f|@2svn=mpq=~vQ_6>!)C1A6Rec5DswUCVwGCUIc z9NA>VZuH|C={=;^sCSFYW|r=1AQP2gH)jAyCc(+s&1D*DZ2RL?(&d@R5lMnFivnmD z%D7^>JaedF0yWF2XBX5&_2hdYDcJj5OuwF;ipIpF8EG_^?lyDU8qKByfzf`BIpb`q zH6x+B9de!}-Jk#gqsO}=p2xR|LB4TneeK-DERK`Ip$OsoU9ZdNEz6j6^lNGKG|83# zEa{Aj7W$qO|9rY`j7+SVk~adPpKr%Y$;m@z!(lxPI&}hZqOxjNno+fm$D%l{@f*)E zA->QsAvI9O>G!Q-IHSOvt2hS#lTIpRI;k@-8;s)cxyv$Ogd_YMQw$yIy#zzgablK3 z{i!%u-9FV_*(T&zvU}8_-hp~|dOOV0y(fXjgGE-qhny&rZNPo`E`0W$T%4-;O$lQF=&BsjZr()MeQ2sR&!ZsgHy#dios zaD3mt?BZlmVhxM7;*}yn+Gc!M;`+~ns=Kt2e<&{&3EP+Fdv7Q=_hru znby-lsHbDe-B4aVM|Vm&B^&+CF|pM$FzT6<@4tbW7b!5kWX-`u9k@o+!IiTs+vk>& zR~H<~4H4j{-e|k-!wXNEP7x!#KR1jSmkk-9iGZ%91P>ze5$*mNh_x<+w*deQ36mxQFeW7MO6o?U*R^4GiHu&4j=bL+pr5$J^qWD(<`K zWqG)xKaL7&Xh_yU6;UUz3yzG}azICab4&?UUF0g*g^^6{cc}AuqEmuzM6K$N0=reRwAtBBL@Bsk+K@MR4VJ8QEzq5w%lMgKC z*E_$jNVYGxci!=KczpYEOqe%iuuiCw_Hmv?*=c82yhm0)gF+a4O*CG(vcV@Dq)4 zC6@EwS}m#(qY5ADt%#Vd9UQ*lxOEw9fzsWtCvh(W*=_F}9FVd>Q+nrMZ)l3Gfq-en zqy`wysTTvVkGOt{15+NXttK}F^RE zh)mSL8&MPM+&UpN_-~U1mhoV~6#B#9=Z^=Bh3u~s;?rbeJn#^Y@lbqG1W!>uvu{y! z)Yf0=JD>HU`~;n+*n-d%44i_9hsSNyFY2E##k7b4QY>3`fdZ)J;3v+%t>| z9~Ih}2?}93Vo#^#iw}=D!O^ifK^rJSa_qse{5i{LM@zuYhIX;U*juvij%7C8F5je_ z!pf<+Sul>UQWG^IaU65o#+B3YsOCejdGB*3%YH4t<`WvE5%O5~IPrJK{#~RhP^9YE zcG$`}!j0cSh8(tq!u;Q6={ zZJdKU@ps3SB0zxj*l!pvV`7#&M@diw)b*QQo}6vX!RMWvAiuJF5FDuAchotKfb4J# z1+c$-orx@Gff`+_WRIW4{=(K}Q)u(d9rpRKEKb2!EZhxgKDAQ=KzlhQp2TJp-eM`gpY5OasQ6}GL zohZC4Oz4JBS)L7>^GDdt+b{U)gH7g>yLm0<`V)iv3d@nZm>+l~ci)LyyH4e(%nc3D z7dmLp95*q}ayr&V)E$jJgP~TAHsk2~UsPTMR6Z%i%50kP{WRDY&7#3hb>*5foO|y@ zCW9GE%^YR~44@ag>@J?LGRPlI6G6w0&;9fH(i`n5|Nq{uJx;3nx_@qJ;!h$-FK8)hyT*WI zVRmPB9=o%Zzyd|7fXXYxjZWrq2L^U`mVF?bv}U7da1~UL*MJJTQrIX#5HV&o)Iv2i zt)xb@6!W2qHfjpAQA6w+`#tA(=g!>O-|XDkUBVyS*?Z6Vp7VR3-|L*Z@A$YAid0s_ zJ-EPJ<%RzpcOXz%8S%hNLX{B@c;gJaNw^<<8vUg=2wHN217#) zlAug zH})j3;Y;Q6tiOv_02xOcoX+9*uBW$Cw!seAYk~#E}!Hv|l z8ZGcXuvmi84v+*pD8WvUu|BYYlx#&4vm1H>Phoi#wRldm8-hY@{Fy{N7a5CwkKvKA zQi{WV54I9Bppp_F_WfvaTao6>%@IX`Vo}F zA~@*!bD5KhvbMGv6lz-EBpwArBp~G3I}&eP?}|u9Q@Wc^E@j7_yr_;0XD%osnH^?0 z-A5t@WDPyAOZ8*2#1#j>*g08RG@ty!O1zH;84iBM{HRqY8HR$4qTKQiC$97!L81Z2 zCh8K4jE1P;f*08q8mz2xV>l%0J((j4%ekNhk-H#BZHnNK#@0F+>pK7!bishz9frSM_;!waW1fiNGpee5aYQTi%7sT30LAZs z9QBBVg+Ts_O0xq=}jY6=VSO^p%SsfbX z2mM0fn8I;?4DI;bh{sV@76b7eGN~`ncx8nZwZ^%OF(_LlwIAX@9zB2uxAR^DW~Q^W zx+@FNmD@kVG+`(te7!_0wAKXA>~KqBu$cyTtv(}$(2uc7Pi1Am6ZN>mLG1YWBY|i| zg$I#4Y1xl}q45Sh)J2bhk-s889sv<*ql}|8=&CS0RN0nS6)gMlSExcnP=#`Na1|lj zoxq{Uc0735kAGqKp%)N3+|jaMaYcbz$M8s+DTS;=2WO$l(GhW=j!4FB6=1xg_Xx8* znP{|~zr$RLS-`mq0N{edHe-2th=Yh2O)UwdITR{yt(K>?Lrv9kwJId-#*le8KGcBo zOv{Z-6S_E@v|X%8U5014oBu zia1(`vvJG`faj$Cz5@Aza|+Fg<`%@(jI=JwWW*!@5dwf-lt8WmKy=0tvD#q($)G+o zHz8JJHr8B8Xm?Y zZwJ6%Q8ixyku@9*x+)A0ReqjV6?6dn6{-*sRH3=r;3`5L0D(i1f5C&7iTD?U(hJ2h z5m~783B@ES`mSz^n!=5X6H)W{?G%)OWN!agV8GIxj5Ot0{Yu-J(t*7T0R!dOgCUZ+ zq2s?XNc*#7uFe~PrRB=dZ%W!+~^m?oO z2qg(8ZEjzcCmIgI<0IM)qeEX3qHZ_v{lQ}#+@U5NCxzwhgu(6^p!*> zOnvAm!8y+NYBt`pspg>+t%CB3x5k!)$4be7opahF2 zeT0hnA?VjENHs;y?71jRu*^q^NNFJ`j6(0@!PkuW7fSWNSp@@zJ2VQ# zNWbvnW9F`|E>O`E4G9EuH=4-1SFfP%@Se>$fcs3sU0VRI&mo*9See0x&WIB#!0+74 zt-R1UXtwg)CP^z`gaDw`63AajOAcFNSr}I1rhbATfV5sB73wGIl8ZuQ5r^;tHby3O zFfSMbRYgKMG%q};5X68Av4K?p-{rQoVjjdxnv8e}Z*gK_7KQ*5Lzd*hIsyu!99wws z3Rzy$h}o!&?ll*VHjLOTb`@DNE{1y(2qN<}8C9EDA$21jyrRs%@M3dXjXqbDQIVMv z^_CbIfq&t}7N-bIiOeSkM&Ms~@sJ|~x~)x=V0fGv(ECt6J4>z9M2E!>MrTGcsIK6$-Cq`Z^$+Q}S)wEfwtBM?x$U)|nZgl_S1LCYUla zKu>3Ui!xz$i2-HzBPV)B5bK7b&LMrw06iV?h0SPnuu+%+dn8x>1i(wq0CEg+nkx|}0A7^=aaRcO(1Su436x%=1+?mZ7pb$Gy9^S`8gF2KqwL>BR-6Krru#=^dCI++%dcY>8 zC79xteVG&8uhZ%rkVg~5(C}%+XSY&`= z^*MXpyVtd2l1}XCCg{O-my?qj=%<+h#U`a6e4k9kJoBuyImY7U?rMiN8zm@3w?8H` za)37`<|8eH1f4o1;NYB!p(;v%C?SdKu!P7CkbLcrvpx>&0IU!)G0H>osLa!%EQ4*Sg0g`^V~D%p;75=>9mGqCy4gtNZ0u(P}Xw}+BV7*)xoC=o*azfaSf zE=A|IHMbb`C1%grhC{RfSkY3_kc=74V1@O;JkXO^J2a9_zwlyb8q1W8%)l#3BZIXCZgw|jDJCyt~7v?x{Y#U1_-wJ3fXT|M~E0aPZT3-BKU{_d`~kU z;yLwg?<_~y$V`teCa7~7_B_fQnDDWwGNA*()4B;s8W;& zWIr=NwLd&(783u$i_h_b1}`oo{)^DNiHh0q3sevAi!!K6zwlxm9z5ptIWZ~)15qMv z7K1sVhwBC(jtRw*^(Ay(6{*7!8>WJMi`aDJF$Xs!remf#FFB<(9yh4N1T>RDmj~0j zRU3}EjVP9h0|@LFqdQksLBH4I!$BS#SY%Xye*$o<%ishk z^;g6#5JA|;|KCdy5!f1oC;(|-WMr%qa4^QTVh}~pNkYO#L4S$|!`n|1UXIeGPe`St z$hkKJ1pa{UpQ#a0)7wvN{EQIj(V<-m7Wtz6)Rxbr(DQ3~^t?HxeObSNd#eU)PN%e? zLd*I43%0*Uia*hZ%UMTtZJ^5U4+bj$@>Ox3&U;Yhs4)#jT{%w8Md>msJ&h21^JyaV zFWS&96JOFtg&UF$mNFvA=O*Xq;d8j9)P2l8D3JbV-A0kps>M*f&>pvf`RKm2S5n3dL&o%Rs3qI*RG_@^@;gC7 zxu|cHY!@I3ltV|~SeVj$Vz{0^;-G@<$(?7;OKCOYPj!c`OB5Jyf4Y5YO1oO(f%&0x?Qg9AdLh z_bWCMtZ%fBpa6dNz@uYQ+V$ek^?n_zR3JY0gnE9a~?dGhP!%@1Fe!cFkM&pFFm^9)|56+T%jwj z5;(jrrO1+xj_pmRu$1!^U2zGNSHUcO6*syRfDavh3j^RS;!0IU6k1lZeb18-YwzlI zQAT`Kl@T@oOY=-Qns+J`dcX7W=U<>+V2r94Z~!YWs49R@{QJuCl=dw#Rv-8YP6hnt zjeCcsv^nA`y}vim6`;TBd2>=qYZBx1bLCA!A@rw*_D@J@bH&&6eqZNOpx%4xOq|+$ zsjAHvu@I+#e*ON#$oiYaczvMGQWW4H9J=p(s*e*?^$`ma*zr`mjIqAuX+Qt*PoJY~ z7YdguLy9<0uh9AI=~E+9+BZa*KA@Fg1@bR;J)xzvZ;En#fWCed*w1`&bV*996>fdN z9*Pv`2cCW&)2CYD84x-r9{2~$e-?-eRV}f6FDN=f>sy{$vijI>U}?-2Ue&lz z&{vj1(-(KWUz38nd!JscA1VddYlmO=Q?mW4WIOhys{p=#>y8UkIKmN7Su_P4p()_* zPJM6z8BmorgK4;^(l6whJKVz2(i2V1g1WU2Z4FnM83KhwrckG&huoBCtbw~xa$mcr zb6wZ>FQT!fS|6NPr#AQ_tJX67B=J|6v!PpwA+CrOsa3}Goyzx%vi#TC5DxB>2$RLt znC7bydPgd@L7Z4GA*P9mn4jcA^c6JO4zpFl+#5u}Zh)H@e zfxc}ZNA8m#GsI-QACb}RSe{Y|gY)OMmaR^f3=>>A)F$pqMOtm$Gp<-Ij z-+!I+_jKFey0}Ik)|hC-Vv}6H@?mnKi)(woEEO~Kk=ct=1~%PkVQTaRxKFvP)deJ8>~bA8famwkEReW!Zmv z#as+ZwC^KNYsGyP{)&h@?uvP$6)vCG7juOJUcbv5fiEpLPFNdJ56wX4!Gg-|k#e1g zmgm06*H`X&QMQt$V!l3Kc}<)4CnjSYNVYcNx)|;sEjA`+2ZfPqbFB$igVBn`+d9{h z;0-g&;!RCSc_ll$^1pn~j;>Rv3`@iU-8?~TM`jHiNeg2&wxPwfVrFz<9${xsOJPgJ zHwKzC^K`MRvTgdDe9}%zX{BOe;YQK9bG_LpzUib<^qAybnz%_{tI1+wZ+&?ye>oRa zxojEbem`!B`MgGDCBg_+_Ue(=QF5YNee&3>+sFxsDL=B z95kXngdGGIf=EZA75+HF)<@%&0e>0OjUG}nXI{y4-lZxz^7;O*J76D_irT_CvGP

sgq2DXnp=q4f#HmiKJedRTW6{mby>{ zB7u5B?iCTA8~-W;aX-A!1w2(L_}Oc8$l&+H4KR4cvVIIfUeVMKu92cSCm%g-SW=~; zMVaE%fx)QY9Fk)q9(6u;YB#Yy>&njzg{>Cud~H3OFJAF?{YdOsahnaks>*F|4Xcjr==uXj?y=%~HbUs( zwdwIDdT5#*A|i^`4@F$!`cp z-3}7pxp6Z$9+$xRBnKvVk4Zm9+@oRF(9Ep4t=MOe8uM`W4qdFv?c#38g`RZ0xYw=) zMcd}95!v;=`<@hcr{l$H2h>}i+Qifsi#1v{%GR@+O_ci_p!94>%e>zK_14`|=6Lad z1M2Qiq$XF1wOV$Is0jFTn$D)zwsX@NA=W9&@IeRCdOnoW=7{y0j+-g4$>o2A3_uuk zElE`Nvc|~h;dFN4`qq}l)|R;~aJyPxvj2^J-=hND;6R1ro6?!N(E)Y$-gIVeazNd) zI*q;A3H3jwu`vlYmVaTiJZQdN66=Lw5b(7)6141 zik4ytdHpYb-%Nw}6a5*)*J$ENO<#2`C#p>b9M`mkj7Tizi}<2gHi?lr95G_Jg&wVp z1tS=rDudg_5q|sDE+6&jn+tu21MHW8oL-yb1k#50b}ff3?>GDDv2kI?br#|~1~Z4DnkaX@|iwCp}di0)pB zZqv@&qqUbD)V_F3vyOVv(N-)%(&>3R0T}STt+@hao-&Wu=Q7tCJwBJm<60C57?u92 UcxB8Tb-TiFs9EW&s`Li`AD? Date: Wed, 26 Nov 2025 17:08:12 +0100 Subject: [PATCH 02/35] redo a seemingly missed spotlessApply --- sentry-android-core/build.gradle.kts | 12 +-- .../android/core/SentryAndroidOptions.java | 5 +- .../android/core/TombstoneIntegration.java | 73 ++++++++--------- .../internal/tombstone/TombstoneParser.java | 59 +++++++------- .../internal/tombstone/TombstoneParserTest.kt | 80 ++++++++++--------- 5 files changed, 111 insertions(+), 118 deletions(-) diff --git a/sentry-android-core/build.gradle.kts b/sentry-android-core/build.gradle.kts index 3fce8891c7a..e293fd76ee9 100644 --- a/sentry-android-core/build.gradle.kts +++ b/sentry-android-core/build.gradle.kts @@ -113,16 +113,8 @@ dependencies { } protobuf { - protoc { - artifact = libs.protoc.get().toString() - } + protoc { artifact = libs.protoc.get().toString() } generateProtoTasks { - all().forEach { task -> - task.builtins { - create("java") { - option("lite") - } - } - } + all().forEach { task -> task.builtins { create("java") { option("lite") } } } } } diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java index 38fe7400c14..23164d644d1 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java @@ -82,7 +82,7 @@ public final class SentryAndroidOptions extends SentryOptions { *

  • The transaction status will be {@link SpanStatus#OK} if none is set. * * - * The transaction is automatically bound to the {@link IScope}, but only if there's no + *

    The transaction is automatically bound to the {@link IScope}, but only if there's no * transaction already bound to the Scope. */ private boolean enableAutoActivityLifecycleTracing = true; @@ -313,7 +313,8 @@ public void setTombstonesEnabled(boolean tombstonesEnabled) { } /** - * Checks if Tombstone reporting (ApplicationExitInfo.REASON_CRASH_NATIVE) is enabled or disabled Default is disabled + * Checks if Tombstone reporting (ApplicationExitInfo.REASON_CRASH_NATIVE) is enabled or disabled + * Default is disabled * * @return true if enabled or false otherwise */ diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java index 2ef46d64869..612704093ca 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -6,9 +6,7 @@ import android.app.ApplicationExitInfo; import android.content.Context; import android.os.Build; - import androidx.annotation.RequiresApi; - import io.sentry.DateUtils; import io.sentry.IScopes; import io.sentry.Integration; @@ -21,13 +19,11 @@ import io.sentry.transport.CurrentDateProvider; import io.sentry.transport.ICurrentDateProvider; import io.sentry.util.Objects; - import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; - import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -45,7 +41,7 @@ public TombstoneIntegration(final @NotNull Context context) { } TombstoneIntegration( - final @NotNull Context context, final @NotNull ICurrentDateProvider dateProvider) { + final @NotNull Context context, final @NotNull ICurrentDateProvider dateProvider) { this.context = ContextUtils.getApplicationContext(context); this.dateProvider = dateProvider; } @@ -53,28 +49,29 @@ public TombstoneIntegration(final @NotNull Context context) { @Override public void register(@NotNull IScopes scopes, @NotNull SentryOptions options) { this.options = - Objects.requireNonNull( - (options instanceof SentryAndroidOptions) ? (SentryAndroidOptions) options : null, - "SentryAndroidOptions is required"); + Objects.requireNonNull( + (options instanceof SentryAndroidOptions) ? (SentryAndroidOptions) options : null, + "SentryAndroidOptions is required"); this.options - .getLogger() - .log(SentryLevel.DEBUG, "TombstoneIntegration enabled: %s", this.options.isTombstonesEnabled()); + .getLogger() + .log( + SentryLevel.DEBUG, + "TombstoneIntegration enabled: %s", + this.options.isTombstonesEnabled()); if (this.options.isTombstonesEnabled()) { if (this.options.getCacheDirPath() == null) { this.options - .getLogger() - .log(SentryLevel.INFO, "Cache dir is not set, unable to process Tombstones"); + .getLogger() + .log(SentryLevel.INFO, "Cache dir is not set, unable to process Tombstones"); return; } try { options - .getExecutorService() - .submit( - new TombstoneProcessor( - context, scopes, this.options, dateProvider)); + .getExecutorService() + .submit(new TombstoneProcessor(context, scopes, this.options, dateProvider)); } catch (Throwable e) { options.getLogger().log(SentryLevel.DEBUG, "Failed to start TombstoneProcessor.", e); } @@ -92,19 +89,16 @@ public void close() throws IOException { public static class TombstoneProcessor implements Runnable { - @NotNull - private final Context context; - @NotNull - private final IScopes scopes; - @NotNull - private final SentryAndroidOptions options; + @NotNull private final Context context; + @NotNull private final IScopes scopes; + @NotNull private final SentryAndroidOptions options; private final long threshold; public TombstoneProcessor( - @NotNull Context context, - @NotNull IScopes scopes, - @NotNull SentryAndroidOptions options, - @NotNull ICurrentDateProvider dateProvider) { + @NotNull Context context, + @NotNull IScopes scopes, + @NotNull SentryAndroidOptions options, + @NotNull ICurrentDateProvider dateProvider) { this.context = context; this.scopes = scopes; this.options = options; @@ -116,7 +110,7 @@ public TombstoneProcessor( @RequiresApi(api = Build.VERSION_CODES.R) public void run() { final ActivityManager activityManager = - (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); + (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); final List applicationExitInfoList; applicationExitInfoList = activityManager.getHistoricalProcessExitReasons(null, 0, 0); @@ -129,12 +123,12 @@ public void run() { final IEnvelopeCache cache = options.getEnvelopeDiskCache(); if (cache instanceof EnvelopeCache) { if (options.isEnableAutoSessionTracking() - && !((EnvelopeCache) cache).waitPreviousSessionFlush()) { + && !((EnvelopeCache) cache).waitPreviousSessionFlush()) { options - .getLogger() - .log( - SentryLevel.WARNING, - "Timed out waiting to flush previous session to its own file."); + .getLogger() + .log( + SentryLevel.WARNING, + "Timed out waiting to flush previous session to its own file."); // if we timed out waiting here, we can already flush the latch, because the timeout is // big @@ -155,7 +149,8 @@ public void run() { if (applicationExitInfo.getReason() == ApplicationExitInfo.REASON_CRASH_NATIVE) { latestTombstone = applicationExitInfo; // remove it, so it's not reported twice - // TODO: if we fail after this, we effectively lost the ApplicationExitInfo (maybe only remove after we reported it) + // TODO: if we fail after this, we effectively lost the ApplicationExitInfo (maybe only + // remove after we reported it) exitInfos.remove(applicationExitInfo); break; } @@ -163,17 +158,17 @@ public void run() { if (latestTombstone == null) { options - .getLogger() - .log( - SentryLevel.DEBUG, - "No Tombstones have been found in the historical exit reasons list."); + .getLogger() + .log( + SentryLevel.DEBUG, + "No Tombstones have been found in the historical exit reasons list."); return; } if (latestTombstone.getTimestamp() < threshold) { options - .getLogger() - .log(SentryLevel.DEBUG, "Latest Tombstones happened too long ago, returning early."); + .getLogger() + .log(SentryLevel.DEBUG, "Latest Tombstones happened too long ago, returning early."); return; } diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index ca1815c42b0..9c4d3407e8d 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -1,15 +1,6 @@ package io.sentry.android.core.internal.tombstone; import androidx.annotation.NonNull; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; - import io.sentry.SentryEvent; import io.sentry.SentryLevel; import io.sentry.protocol.DebugImage; @@ -20,6 +11,13 @@ import io.sentry.protocol.SentryStackFrame; import io.sentry.protocol.SentryStackTrace; import io.sentry.protocol.SentryThread; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; public class TombstoneParser { @@ -44,7 +42,7 @@ public SentryEvent parse() throws IOException { SentryEvent event = new SentryEvent(); event.setLevel(SentryLevel.FATAL); - // we must use the "native" platform because otherwise the stack-trace would not be correctly parsed + // must use the "native" platform because otherwise the stack-trace wouldn't be correctly parsed event.setPlatform("native"); event.setMessage(constructMessage(tombstone)); @@ -57,10 +55,11 @@ public SentryEvent parse() throws IOException { } @NonNull - private List createThreads(TombstoneProtos.Tombstone tombstone, SentryException exc) { + private List createThreads( + TombstoneProtos.Tombstone tombstone, SentryException exc) { List threads = new ArrayList<>(); for (Map.Entry threadEntry : - tombstone.getThreadsMap().entrySet()) { + tombstone.getThreadsMap().entrySet()) { SentryThread thread = new SentryThread(); thread.setId(Long.valueOf(threadEntry.getKey())); @@ -70,7 +69,8 @@ private List createThreads(TombstoneProtos.Tombstone tombstone, Se thread.setStacktrace(stacktrace); if (tombstone.getTid() == threadEntry.getValue().getId()) { thread.setCrashed(true); - // even though we refer to the thread_id from the exception, the backend currently requires a stack-trace in exception + // even though we refer to the thread_id from the exception, + // the backend currently requires a stack-trace in exception exc.setStacktrace(stacktrace); } threads.add(thread); @@ -80,11 +80,11 @@ private List createThreads(TombstoneProtos.Tombstone tombstone, Se } @NonNull - private static SentryStackTrace createStackTrace(Map.Entry threadEntry) { + private static SentryStackTrace createStackTrace( + Map.Entry threadEntry) { List frames = new ArrayList<>(); - for (TombstoneProtos.BacktraceFrame frame : - threadEntry.getValue().getCurrentBacktraceList()) { + for (TombstoneProtos.BacktraceFrame frame : threadEntry.getValue().getCurrentBacktraceList()) { SentryStackFrame stackFrame = new SentryStackFrame(); stackFrame.setPackage(frame.getFileName()); stackFrame.setFunction(frame.getFunctionName()); @@ -96,7 +96,7 @@ private static SentryStackTrace createStackTrace(Map.Entry unknown = new HashMap<>(); - // `libunwindstack` used for tombstone generation already applies instruction address adjustment: + // `libunwindstack` used for tombstones already applies instruction address adjustment: // https://android.googlesource.com/platform/system/unwinding/+/refs/heads/main/libunwindstack/Regs.cpp#175 // prevent "processing" from doing it again. unknown.put("instruction_addr_adjustment", "none"); @@ -136,8 +136,8 @@ private static Mechanism createMechanismFromSignalInfo(TombstoneProtos.Signal si meta.put("code_name", signalInfo.getCodeName()); Mechanism mechanism = new Mechanism(); - // this follows the current processing triggers strictly, changing any of these alters grouping and name (long-term we might want to - // have a tombstone mechanism) + // this follows the current processing triggers strictly, changing any of these + // alters grouping and name (long-term we might want to have a tombstone mechanism) mechanism.setType("signalhandler"); mechanism.setHandled(false); mechanism.setSynthetic(true); @@ -153,14 +153,15 @@ private Message constructMessage(TombstoneProtos.Tombstone tombstone) { // reproduce the message `debuggerd` would use to dump the stack trace in logcat message.setFormatted( - String.format(Locale.getDefault(), - "Fatal signal %s (%d), %s (%d), pid = %d (%s)", - signalInfo.getName(), - signalInfo.getNumber(), - signalInfo.getCodeName(), - signalInfo.getCode(), - tombstone.getPid(), - String.join(" ", tombstone.getCommandLineList()))); + String.format( + Locale.getDefault(), + "Fatal signal %s (%d), %s (%d), pid = %d (%s)", + signalInfo.getName(), + signalInfo.getNumber(), + signalInfo.getCodeName(), + signalInfo.getCode(), + tombstone.getPid(), + String.join(" ", tombstone.getCommandLineList()))); return message; } @@ -171,8 +172,8 @@ private DebugMeta createDebugMeta(TombstoneProtos.Tombstone tombstone) { for (TombstoneProtos.MemoryMapping module : tombstone.getMemoryMappingsList()) { // exclude anonymous and non-executable maps if (module.getBuildId().isEmpty() - || module.getMappingName().isEmpty() - || !module.getExecute()) { + || module.getMappingName().isEmpty() + || !module.getExecute()) { continue; } DebugImage image = new DebugImage(); diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/internal/tombstone/TombstoneParserTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/internal/tombstone/TombstoneParserTest.kt index 3f321dc4d80..f4e17f04565 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/internal/tombstone/TombstoneParserTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/internal/tombstone/TombstoneParserTest.kt @@ -6,43 +6,44 @@ import kotlin.test.assertEquals import kotlin.test.assertNotNull class TombstoneParserTest { - val expectedRegisters = setOf( - "x8", - "x9", - "esr", - "lr", - "pst", - "x10", - "x12", - "x11", - "x14", - "x13", - "x16", - "x15", - "sp", - "x18", - "x17", - "x19", - "pc", - "x21", - "x20", - "x0", - "x23", - "x1", - "x22", - "x2", - "x25", - "x3", - "x24", - "x4", - "x27", - "x5", - "x26", - "x6", - "x29", - "x7", - "x28" - ) + val expectedRegisters = + setOf( + "x8", + "x9", + "esr", + "lr", + "pst", + "x10", + "x12", + "x11", + "x14", + "x13", + "x16", + "x15", + "sp", + "x18", + "x17", + "x19", + "pc", + "x21", + "x20", + "x0", + "x23", + "x1", + "x22", + "x2", + "x25", + "x3", + "x24", + "x4", + "x27", + "x5", + "x26", + "x6", + "x29", + "x7", + "x28", + ) @Test fun `parses a snapshot tombstone into Event`() { @@ -52,7 +53,10 @@ class TombstoneParserTest { // top-level data assertNotNull(event.eventId) - assertEquals("Fatal signal SIGSEGV (11), SEGV_MAPERR (1), pid = 21891 (io.sentry.samples.android)", event.message!!.formatted) + assertEquals( + "Fatal signal SIGSEGV (11), SEGV_MAPERR (1), pid = 21891 (io.sentry.samples.android)", + event.message!!.formatted, + ) assertEquals("native", event.platform) assertEquals("FATAL", event.level!!.name) From 10b1e9483125af238d17e1bc793fd5fced93ad7d Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Wed, 26 Nov 2025 19:36:16 +0100 Subject: [PATCH 03/35] use non-plural option + mark integration also as internal + update API --- sentry-android-core/api/sentry-android-core.api | 13 +++++++++++++ .../sentry/android/core/SentryAndroidOptions.java | 12 ++++++------ .../sentry/android/core/TombstoneIntegration.java | 7 +++++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/sentry-android-core/api/sentry-android-core.api b/sentry-android-core/api/sentry-android-core.api index 1c89d8524c0..6efbc5d4f70 100644 --- a/sentry-android-core/api/sentry-android-core.api +++ b/sentry-android-core/api/sentry-android-core.api @@ -339,6 +339,7 @@ public final class io/sentry/android/core/SentryAndroidOptions : io/sentry/Sentr public fun isEnableSystemEventBreadcrumbs ()Z public fun isEnableSystemEventBreadcrumbsExtras ()Z public fun isReportHistoricalAnrs ()Z + public fun isTombstoneEnabled ()Z public fun setAnrEnabled (Z)V public fun setAnrReportInDebug (Z)V public fun setAnrTimeoutIntervalMillis (J)V @@ -367,6 +368,7 @@ public final class io/sentry/android/core/SentryAndroidOptions : io/sentry/Sentr public fun setNativeHandlerStrategy (Lio/sentry/android/core/NdkHandlerStrategy;)V public fun setNativeSdkName (Ljava/lang/String;)V public fun setReportHistoricalAnrs (Z)V + public fun setTombstoneEnabled (Z)V } public abstract interface class io/sentry/android/core/SentryAndroidOptions$BeforeCaptureCallback { @@ -455,6 +457,17 @@ public final class io/sentry/android/core/SystemEventsBreadcrumbsIntegration : i public fun register (Lio/sentry/IScopes;Lio/sentry/SentryOptions;)V } +public class io/sentry/android/core/TombstoneIntegration : io/sentry/Integration, java/io/Closeable { + public fun (Landroid/content/Context;)V + public fun close ()V + public fun register (Lio/sentry/IScopes;Lio/sentry/SentryOptions;)V +} + +public class io/sentry/android/core/TombstoneIntegration$TombstoneProcessor : java/lang/Runnable { + public fun (Landroid/content/Context;Lio/sentry/IScopes;Lio/sentry/android/core/SentryAndroidOptions;Lio/sentry/transport/ICurrentDateProvider;)V + public fun run ()V +} + public final class io/sentry/android/core/UserInteractionIntegration : android/app/Application$ActivityLifecycleCallbacks, io/sentry/Integration, java/io/Closeable { public fun (Landroid/app/Application;Lio/sentry/util/LoadClass;)V public fun close ()V diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java index 23164d644d1..5f21444933d 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java @@ -227,7 +227,7 @@ public interface BeforeCaptureCallback { private @Nullable SentryFrameMetricsCollector frameMetricsCollector; - private boolean tombstonesEnabled = false; + private boolean enableTombstone = false; public SentryAndroidOptions() { setSentryClientName(BuildConfig.SENTRY_ANDROID_SDK_NAME + "/" + BuildConfig.VERSION_NAME); @@ -305,11 +305,11 @@ public void setAnrReportInDebug(boolean anrReportInDebug) { /** * Sets Tombstone reporting (ApplicationExitInfo.REASON_CRASH_NATIVE) to enabled or disabled. * - * @param tombstonesEnabled true for enabled and false for disabled + * @param enableTombstone true for enabled and false for disabled */ @ApiStatus.Internal - public void setTombstonesEnabled(boolean tombstonesEnabled) { - this.tombstonesEnabled = tombstonesEnabled; + public void setTombstoneEnabled(boolean enableTombstone) { + this.enableTombstone = enableTombstone; } /** @@ -319,8 +319,8 @@ public void setTombstonesEnabled(boolean tombstonesEnabled) { * @return true if enabled or false otherwise */ @ApiStatus.Internal - public boolean isTombstonesEnabled() { - return tombstonesEnabled; + public boolean isTombstoneEnabled() { + return enableTombstone; } public boolean isEnableActivityLifecycleBreadcrumbs() { diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java index 612704093ca..0b20553495f 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -24,9 +24,11 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +@ApiStatus.Internal public class TombstoneIntegration implements Integration, Closeable { static final long NINETY_DAYS_THRESHOLD = TimeUnit.DAYS.toMillis(91); @@ -58,9 +60,9 @@ public void register(@NotNull IScopes scopes, @NotNull SentryOptions options) { .log( SentryLevel.DEBUG, "TombstoneIntegration enabled: %s", - this.options.isTombstonesEnabled()); + this.options.isTombstoneEnabled()); - if (this.options.isTombstonesEnabled()) { + if (this.options.isTombstoneEnabled()) { if (this.options.getCacheDirPath() == null) { this.options .getLogger() @@ -87,6 +89,7 @@ public void close() throws IOException { } } + @ApiStatus.Internal public static class TombstoneProcessor implements Runnable { @NotNull private final Context context; From 4b6e1fbe3cea3b90052d8ddfe04cd1b633c0a41d Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Thu, 27 Nov 2025 11:01:11 +0100 Subject: [PATCH 04/35] init size exceptions since we know we only need one Co-authored-by: Markus Hintersteiner --- .../sentry/android/core/internal/tombstone/TombstoneParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index 9c4d3407e8d..f1efc4bfaa6 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -121,7 +121,7 @@ private List createException(TombstoneProtos.Tombstone tombston exception.setMechanism(createMechanismFromSignalInfo(signalInfo)); exception.setThreadId((long) tombstone.getTid()); - List exceptions = new ArrayList<>(); + List exceptions = new ArrayList<>(1); exceptions.add(exception); return exceptions; From 95bd72608cd6ffdd96c2fa7cf23e5e12701d0831 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 16:20:55 +0100 Subject: [PATCH 05/35] move `instructionAddressAdjustment` to `SentryStackTrace` --- .../internal/tombstone/TombstoneParser.java | 49 ++++++++++--------- .../io/sentry/protocol/SentryStackTrace.java | 24 +++++++++ 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index f1efc4bfaa6..8e47cbfa995 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -60,14 +60,15 @@ private List createThreads( List threads = new ArrayList<>(); for (Map.Entry threadEntry : tombstone.getThreadsMap().entrySet()) { + TombstoneProtos.Thread threadEntryValue = threadEntry.getValue(); SentryThread thread = new SentryThread(); thread.setId(Long.valueOf(threadEntry.getKey())); - thread.setName(threadEntry.getValue().getName()); + thread.setName(threadEntryValue.getName()); - SentryStackTrace stacktrace = createStackTrace(threadEntry); + SentryStackTrace stacktrace = createStackTrace(threadEntryValue); thread.setStacktrace(stacktrace); - if (tombstone.getTid() == threadEntry.getValue().getId()) { + if (tombstone.getTid() == threadEntryValue.getId()) { thread.setCrashed(true); // even though we refer to the thread_id from the exception, // the backend currently requires a stack-trace in exception @@ -80,11 +81,10 @@ private List createThreads( } @NonNull - private static SentryStackTrace createStackTrace( - Map.Entry threadEntry) { + private static SentryStackTrace createStackTrace(TombstoneProtos.Thread thread) { List frames = new ArrayList<>(); - for (TombstoneProtos.BacktraceFrame frame : threadEntry.getValue().getCurrentBacktraceList()) { + for (TombstoneProtos.BacktraceFrame frame : thread.getCurrentBacktraceList()) { SentryStackFrame stackFrame = new SentryStackFrame(); stackFrame.setPackage(frame.getFileName()); stackFrame.setFunction(frame.getFunctionName()); @@ -95,15 +95,13 @@ private static SentryStackTrace createStackTrace( SentryStackTrace stacktrace = new SentryStackTrace(); stacktrace.setFrames(frames); - Map unknown = new HashMap<>(); // `libunwindstack` used for tombstones already applies instruction address adjustment: // https://android.googlesource.com/platform/system/unwinding/+/refs/heads/main/libunwindstack/Regs.cpp#175 // prevent "processing" from doing it again. - unknown.put("instruction_addr_adjustment", "none"); - stacktrace.setUnknown(unknown); + stacktrace.setInstructionAddressAdjustment("none"); Map registers = new HashMap<>(); - for (TombstoneProtos.Register register : threadEntry.getValue().getRegistersList()) { + for (TombstoneProtos.Register register : thread.getRegistersList()) { registers.put(register.getName(), String.format("0x%x", register.getU64())); } stacktrace.setRegisters(registers); @@ -113,14 +111,16 @@ private static SentryStackTrace createStackTrace( @NonNull private List createException(TombstoneProtos.Tombstone tombstone) { - TombstoneProtos.Signal signalInfo = tombstone.getSignalInfo(); - SentryException exception = new SentryException(); - exception.setType(signalInfo.getName()); - exception.setValue(excTypeValueMap.get(signalInfo.getName())); - exception.setMechanism(createMechanismFromSignalInfo(signalInfo)); - exception.setThreadId((long) tombstone.getTid()); + if (tombstone.hasSignalInfo()) { + TombstoneProtos.Signal signalInfo = tombstone.getSignalInfo(); + exception.setType(signalInfo.getName()); + exception.setValue(excTypeValueMap.get(signalInfo.getName())); + exception.setMechanism(createMechanismFromSignalInfo(signalInfo)); + } + + exception.setThreadId((long) tombstone.getTid()); List exceptions = new ArrayList<>(1); exceptions.add(exception); @@ -129,18 +129,23 @@ private List createException(TombstoneProtos.Tombstone tombston @NonNull private static Mechanism createMechanismFromSignalInfo(TombstoneProtos.Signal signalInfo) { - Map meta = new HashMap<>(); - meta.put("number", signalInfo.getNumber()); - meta.put("name", signalInfo.getName()); - meta.put("code", signalInfo.getCode()); - meta.put("code_name", signalInfo.getCodeName()); Mechanism mechanism = new Mechanism(); // this follows the current processing triggers strictly, changing any of these // alters grouping and name (long-term we might want to have a tombstone mechanism) + // TODO: if we align this with ANRv2 this would be overwritten in a BackfillingEventProcessor as + // `ApplicationExitInfo` not sure what the right call is. `ApplicationExitInfo` is certainly correct. But `signalhandler` isn't + // wrong either, since all native crashes retrieved via `REASON_CRASH_NATIVE` will be signals. I am not sure what the side-effect + // in ingestion/processing will be if we change the mechanism, but initially i wanted to stay close to the Native SDK. mechanism.setType("signalhandler"); mechanism.setHandled(false); mechanism.setSynthetic(true); + + Map meta = new HashMap<>(); + meta.put("number", signalInfo.getNumber()); + meta.put("name", signalInfo.getName()); + meta.put("code", signalInfo.getCode()); + meta.put("code_name", signalInfo.getCodeName()); mechanism.setMeta(meta); return mechanism; @@ -154,7 +159,7 @@ private Message constructMessage(TombstoneProtos.Tombstone tombstone) { // reproduce the message `debuggerd` would use to dump the stack trace in logcat message.setFormatted( String.format( - Locale.getDefault(), + Locale.ROOT, "Fatal signal %s (%d), %s (%d), pid = %d (%s)", signalInfo.getName(), signalInfo.getNumber(), diff --git a/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java b/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java index c635f78673b..ff61bab87d2 100644 --- a/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java +++ b/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java @@ -66,6 +66,15 @@ public final class SentryStackTrace implements JsonUnknown, JsonSerializable { */ private @Nullable Boolean snapshot; + /** + * This value indicates if, and how, `instruction_addr` values in the stack frames need to be adjusted before they are symbolicated. + * TODO: should we make this an enum or is a string value fine? + * + * @see SentryStackFrame#getInstructionAddr() + * @see SentryStackFrame#setInstructionAddr(String) + */ + private @Nullable String instructionAddressAdjustment; + @SuppressWarnings("unused") private @Nullable Map unknown; @@ -122,10 +131,19 @@ public void setUnknown(@Nullable Map unknown) { this.unknown = unknown; } + public @Nullable String getInstructionAddressAdjustment() { + return instructionAddressAdjustment; + } + + public void setInstructionAddressAdjustment(@Nullable String instructionAddressAdjustment) { + this.instructionAddressAdjustment = instructionAddressAdjustment; + } + public static final class JsonKeys { public static final String FRAMES = "frames"; public static final String REGISTERS = "registers"; public static final String SNAPSHOT = "snapshot"; + public static final String INSTRUCTION_ADDRESS_ADJUSTMENT = "instruction_add_adjustment"; } @Override @@ -141,6 +159,9 @@ public void serialize(final @NotNull ObjectWriter writer, final @NotNull ILogger if (snapshot != null) { writer.name(JsonKeys.SNAPSHOT).value(snapshot); } + if (instructionAddressAdjustment != null) { + writer.name(JsonKeys.INSTRUCTION_ADDRESS_ADJUSTMENT).value(instructionAddressAdjustment); + } if (unknown != null) { for (String key : unknown.keySet()) { Object value = unknown.get(key); @@ -174,6 +195,9 @@ public static final class Deserializer implements JsonDeserializer(); From 4f03857a592bf8679043496ecd1d2aa74e7f6e87 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 16:21:19 +0100 Subject: [PATCH 06/35] add copyright notice to tombstone.proto --- .../core/internal/tombstone/tombstone.proto | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sentry-android-core/src/main/proto/io/sentry/android/core/internal/tombstone/tombstone.proto b/sentry-android-core/src/main/proto/io/sentry/android/core/internal/tombstone/tombstone.proto index c75eae32688..2f9cbe52850 100644 --- a/sentry-android-core/src/main/proto/io/sentry/android/core/internal/tombstone/tombstone.proto +++ b/sentry-android-core/src/main/proto/io/sentry/android/core/internal/tombstone/tombstone.proto @@ -2,6 +2,20 @@ // Sentry changes: // * change the java_package // +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// // Protobuf definition for Android tombstones. // // An app can get hold of these for any `REASON_CRASH_NATIVE` instance of From 936a2f0c8828fa2810ca4348f00b77f27bfba487 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 16:22:06 +0100 Subject: [PATCH 07/35] add historical tombstone option --- .../android/core/SentryAndroidOptions.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java index 5f21444933d..c84c26a565a 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java @@ -217,6 +217,17 @@ public interface BeforeCaptureCallback { */ private boolean reportHistoricalAnrs = false; + /** + * Controls whether to report historical Tombstones from the {@link ApplicationExitInfo} system + * API. When enabled, reports all of the Tombstones available in the {@link + * ActivityManager#getHistoricalProcessExitReasons(String, int, int)} list, as opposed to + * reporting only the latest one. + * + *

    These events do not affect crash rate nor are they enriched with additional information from + * {@link IScope} like breadcrumbs. + */ + private boolean reportHistoricalTombstones = false; + /** * Controls whether to send ANR (v2) thread dump as an attachment with plain text. The thread dump * is being attached from {@link ApplicationExitInfo#getTraceInputStream()}, if available. @@ -593,6 +604,14 @@ public void setReportHistoricalAnrs(final boolean reportHistoricalAnrs) { this.reportHistoricalAnrs = reportHistoricalAnrs; } + public boolean isReportHistoricalTombstones() { + return reportHistoricalTombstones; + } + + public void setReportHistoricalTombstones(final boolean reportHistoricalTombstones) { + this.reportHistoricalTombstones = reportHistoricalTombstones; + } + public boolean isAttachAnrThreadDump() { return attachAnrThreadDump; } From 28652e0a6939abe9eb6fe75877e3dd04f25d7c21 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 16:25:13 +0100 Subject: [PATCH 08/35] looks like we need to add a TombstoneEventProcessor anyway. --- .../api/sentry-android-core.api | 4 +++ .../android/core/TombstoneEventProcessor.java | 26 +++++++++++++++++++ .../internal/tombstone/TombstoneParser.java | 8 +++--- 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 sentry-android-core/src/main/java/io/sentry/android/core/TombstoneEventProcessor.java diff --git a/sentry-android-core/api/sentry-android-core.api b/sentry-android-core/api/sentry-android-core.api index 6efbc5d4f70..901f1fddc25 100644 --- a/sentry-android-core/api/sentry-android-core.api +++ b/sentry-android-core/api/sentry-android-core.api @@ -457,6 +457,10 @@ public final class io/sentry/android/core/SystemEventsBreadcrumbsIntegration : i public fun register (Lio/sentry/IScopes;Lio/sentry/SentryOptions;)V } +public class io/sentry/android/core/TombstoneEventProcessor : io/sentry/BackfillingEventProcessor { + public fun (Landroid/content/Context;Lio/sentry/android/core/SentryAndroidOptions;Lio/sentry/android/core/BuildInfoProvider;)V +} + public class io/sentry/android/core/TombstoneIntegration : io/sentry/Integration, java/io/Closeable { public fun (Landroid/content/Context;)V public fun close ()V diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneEventProcessor.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneEventProcessor.java new file mode 100644 index 00000000000..ef064e13705 --- /dev/null +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneEventProcessor.java @@ -0,0 +1,26 @@ +package io.sentry.android.core; + +import android.content.Context; +import androidx.annotation.WorkerThread; +import io.sentry.BackfillingEventProcessor; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; + +/** originating from the Native SDK. */ +@ApiStatus.Internal +@WorkerThread +public class TombstoneEventProcessor implements BackfillingEventProcessor { + + @NotNull private final Context context; + @NotNull private final SentryAndroidOptions options; + @NotNull private final BuildInfoProvider buildInfoProvider; + + public TombstoneEventProcessor( + @NotNull Context context, + @NotNull SentryAndroidOptions options, + @NotNull BuildInfoProvider buildInfoProvider) { + this.context = context; + this.options = options; + this.buildInfoProvider = buildInfoProvider; + } +} diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index 8e47cbfa995..5e435bf5971 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -134,9 +134,11 @@ private static Mechanism createMechanismFromSignalInfo(TombstoneProtos.Signal si // this follows the current processing triggers strictly, changing any of these // alters grouping and name (long-term we might want to have a tombstone mechanism) // TODO: if we align this with ANRv2 this would be overwritten in a BackfillingEventProcessor as - // `ApplicationExitInfo` not sure what the right call is. `ApplicationExitInfo` is certainly correct. But `signalhandler` isn't - // wrong either, since all native crashes retrieved via `REASON_CRASH_NATIVE` will be signals. I am not sure what the side-effect - // in ingestion/processing will be if we change the mechanism, but initially i wanted to stay close to the Native SDK. + // `ApplicationExitInfo` not sure what the right call is. `ApplicationExitInfo` is + // certainly correct. But `signalhandler` isn't wrong either, since all native crashes + // retrieved via `REASON_CRASH_NATIVE` will be signals. I am not sure what the side-effect + // in ingestion/processing will be if we change the mechanism, but initially i wanted to + // stay close to the Native SDK. mechanism.setType("signalhandler"); mechanism.setHandled(false); mechanism.setSynthetic(true); From 920221a4f672644c8d57bd17b06c0450b6531ade Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 16:30:40 +0100 Subject: [PATCH 09/35] ...so let's add it to the options if the SDK supports it (adapt to >= S, since that is the earliest version where `REASON_CRASH_NATIVE` provides tombstones via the `traceInputStream`) --- .../io/sentry/android/core/AndroidOptionsInitializer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java index ef60b406ecd..f0ce04d0880 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java @@ -189,6 +189,9 @@ static void initializeIntegrationsAndProcessors( options.addEventProcessor(new ScreenshotEventProcessor(options, buildInfoProvider)); options.addEventProcessor(new ViewHierarchyEventProcessor(options)); options.addEventProcessor(new AnrV2EventProcessor(context, options, buildInfoProvider)); + if (buildInfoProvider.getSdkInfoVersion() >= Build.VERSION_CODES.S) { + options.addEventProcessor(new TombstoneEventProcessor(context, options, buildInfoProvider)); + } if (options.getTransportGate() instanceof NoOpTransportGate) { options.setTransportGate(new AndroidTransportGate(options)); } @@ -373,7 +376,7 @@ static void installDefaultIntegrations( final Class sentryNdkClass = loadClass.loadClass(SENTRY_NDK_CLASS_NAME, options.getLogger()); options.addIntegration(new NdkIntegration(sentryNdkClass)); - if (buildInfoProvider.getSdkInfoVersion() >= Build.VERSION_CODES.R) { + if (buildInfoProvider.getSdkInfoVersion() >= Build.VERSION_CODES.S) { options.addIntegration(new TombstoneIntegration(context)); } From 48ed5e123c3cfbbfa31c012990b7588c52197b5e Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 16:32:32 +0100 Subject: [PATCH 10/35] Adapt `AndroidEnvelopeCache` to also write Tombstone timestamp markers --- .../api/sentry-android-core.api | 2 + .../core/cache/AndroidEnvelopeCache.java | 70 ++++++++++++++++--- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/sentry-android-core/api/sentry-android-core.api b/sentry-android-core/api/sentry-android-core.api index 901f1fddc25..ad3d0fcd38e 100644 --- a/sentry-android-core/api/sentry-android-core.api +++ b/sentry-android-core/api/sentry-android-core.api @@ -499,10 +499,12 @@ public final class io/sentry/android/core/ViewHierarchyEventProcessor : io/sentr public final class io/sentry/android/core/cache/AndroidEnvelopeCache : io/sentry/cache/EnvelopeCache { public static final field LAST_ANR_REPORT Ljava/lang/String; + public static final field LAST_TOMBSTONE_REPORT Ljava/lang/String; public fun (Lio/sentry/android/core/SentryAndroidOptions;)V public fun getDirectory ()Ljava/io/File; public static fun hasStartupCrashMarker (Lio/sentry/SentryOptions;)Z public static fun lastReportedAnr (Lio/sentry/SentryOptions;)Ljava/lang/Long; + public static fun lastReportedTombstone (Lio/sentry/SentryOptions;)Ljava/lang/Long; public fun store (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V public fun storeEnvelope (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)Z } diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java b/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java index 6481be70b0e..b8727dad0c6 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java @@ -10,6 +10,7 @@ import io.sentry.UncaughtExceptionHandlerIntegration; import io.sentry.android.core.AnrV2Integration; import io.sentry.android.core.SentryAndroidOptions; +import io.sentry.android.core.TombstoneIntegration; import io.sentry.android.core.internal.util.AndroidCurrentDateProvider; import io.sentry.android.core.performance.AppStartMetrics; import io.sentry.android.core.performance.TimeSpan; @@ -31,6 +32,7 @@ public final class AndroidEnvelopeCache extends EnvelopeCache { public static final String LAST_ANR_REPORT = "last_anr_report"; + public static final String LAST_TOMBSTONE_REPORT = "last_tombstone_report"; private final @NotNull ICurrentDateProvider currentDateProvider; @@ -94,6 +96,22 @@ private boolean storeInternalAndroid(@NotNull SentryEnvelope envelope, @NotNull writeLastReportedAnrMarker(timestamp); }); + + HintUtils.runIfHasType( + hint, + TombstoneIntegration.TombstoneHint.class, + (tombstoneHint) -> { + final @Nullable Long timestamp = tombstoneHint.timestamp(); + options + .getLogger() + .log( + SentryLevel.DEBUG, + "Writing last reported Tombstone marker with timestamp %d", + timestamp); + + writeLastReportedTombstoneMarker(timestamp); + }); + return didStore; } @@ -152,14 +170,18 @@ public static boolean hasStartupCrashMarker(final @NotNull SentryOptions options return false; } - public static @Nullable Long lastReportedAnr(final @NotNull SentryOptions options) { + private static @Nullable Long lastReportedMarker( + final @NotNull SentryOptions options, + @NotNull String reportFilename, + @NotNull String markerCategory) { final String cacheDirPath = Objects.requireNonNull( - options.getCacheDirPath(), "Cache dir path should be set for getting ANRs reported"); + options.getCacheDirPath(), + "Cache dir path should be set for getting " + markerCategory + "s reported"); - final File lastAnrMarker = new File(cacheDirPath, LAST_ANR_REPORT); + final File lastMarker = new File(cacheDirPath, reportFilename); try { - final String content = FileUtils.readText(lastAnrMarker); + final String content = FileUtils.readText(lastMarker); // we wrapped into try-catch already //noinspection ConstantConditions return content.equals("null") ? null : Long.parseLong(content.trim()); @@ -167,27 +189,55 @@ public static boolean hasStartupCrashMarker(final @NotNull SentryOptions options if (e instanceof FileNotFoundException) { options .getLogger() - .log(DEBUG, "Last ANR marker does not exist. %s.", lastAnrMarker.getAbsolutePath()); + .log( + DEBUG, + "Last " + markerCategory + " marker does not exist. %s.", + lastMarker.getAbsolutePath()); } else { - options.getLogger().log(ERROR, "Error reading last ANR marker", e); + options.getLogger().log(ERROR, "Error reading last " + markerCategory + " marker", e); } } return null; } - private void writeLastReportedAnrMarker(final @Nullable Long timestamp) { + private void writeLastReportedMarker( + final @Nullable Long timestamp, + @NotNull String reportFilename, + @NotNull String markerCategory) { final String cacheDirPath = options.getCacheDirPath(); if (cacheDirPath == null) { - options.getLogger().log(DEBUG, "Cache dir path is null, the ANR marker will not be written"); + options + .getLogger() + .log( + DEBUG, + "Cache dir path is null, the " + markerCategory + " marker will not be written"); return; } - final File anrMarker = new File(cacheDirPath, LAST_ANR_REPORT); + final File anrMarker = new File(cacheDirPath, reportFilename); try (final OutputStream outputStream = new FileOutputStream(anrMarker)) { outputStream.write(String.valueOf(timestamp).getBytes(UTF_8)); outputStream.flush(); } catch (Throwable e) { - options.getLogger().log(ERROR, "Error writing the ANR marker to the disk", e); + options + .getLogger() + .log(ERROR, "Error writing the " + markerCategory + " marker to the disk", e); } } + + public static @Nullable Long lastReportedAnr(final @NotNull SentryOptions options) { + return lastReportedMarker(options, LAST_ANR_REPORT, "ANR"); + } + + private void writeLastReportedAnrMarker(final @Nullable Long timestamp) { + this.writeLastReportedMarker(timestamp, LAST_ANR_REPORT, "ANR"); + } + + public static @Nullable Long lastReportedTombstone(final @NotNull SentryOptions options) { + return lastReportedMarker(options, LAST_TOMBSTONE_REPORT, "Tombstone"); + } + + private void writeLastReportedTombstoneMarker(final @Nullable Long timestamp) { + this.writeLastReportedMarker(timestamp, LAST_TOMBSTONE_REPORT, "Tombstone"); + } } From 10c7a1f3de49f160f78abee92b1aa33b5e3fb646 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 16:59:34 +0100 Subject: [PATCH 11/35] Integrate handling of historical Tombstone option + last tombstone marker This currently does not work: While we now can optionally enable reporting of "historical" tombstones, by making the `TombstoneHint` `Backfillable` it will automatically be enriched by the `ANRv2EventProcessor` which is currently the only `BackfillingEventProcessor`. The `ANRv2EventProcessor` is partially written in way that is potentially generic for events with `Backfillable` hints, but other parts are enriching as if those are always were ANRs, which up to now was true, but with Tombstones that assumption now breaks. Next Steps: * There is considerable duplication between the ANRv2Integration and TombstoneIntegration --- .../android/core/TombstoneIntegration.java | 164 ++++++++++++++++-- .../java/io/sentry/hints/NativeCrashExit.java | 15 ++ 2 files changed, 168 insertions(+), 11 deletions(-) create mode 100644 sentry/src/main/java/io/sentry/hints/NativeCrashExit.java diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java index 0b20553495f..225f7f3df82 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -8,20 +8,32 @@ import android.os.Build; import androidx.annotation.RequiresApi; import io.sentry.DateUtils; +import io.sentry.Hint; +import io.sentry.ILogger; import io.sentry.IScopes; import io.sentry.Integration; import io.sentry.SentryEvent; import io.sentry.SentryLevel; import io.sentry.SentryOptions; +import io.sentry.android.core.cache.AndroidEnvelopeCache; import io.sentry.android.core.internal.tombstone.TombstoneParser; import io.sentry.cache.EnvelopeCache; import io.sentry.cache.IEnvelopeCache; +import io.sentry.hints.Backfillable; +import io.sentry.hints.BlockingFlushHint; +import io.sentry.hints.NativeCrashExit; +import io.sentry.protocol.SentryId; import io.sentry.transport.CurrentDateProvider; import io.sentry.transport.ICurrentDateProvider; +import io.sentry.util.HintUtils; import io.sentry.util.Objects; import java.io.Closeable; import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import org.jetbrains.annotations.ApiStatus; @@ -115,6 +127,11 @@ public void run() { final ActivityManager activityManager = (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); + if (activityManager == null) { + options.getLogger().log(SentryLevel.ERROR, "Failed to retrieve ActivityManager."); + return; + } + final List applicationExitInfoList; applicationExitInfoList = activityManager.getHistoricalProcessExitReasons(null, 0, 0); @@ -134,8 +151,7 @@ public void run() { "Timed out waiting to flush previous session to its own file."); // if we timed out waiting here, we can already flush the latch, because the timeout is - // big - // enough to wait for it only once and we don't have to wait again in + // big enough to wait for it only once and we don't have to wait again in // PreviousSessionFinalizer ((EnvelopeCache) cache).flushPreviousSession(); } @@ -143,6 +159,8 @@ public void run() { // making a deep copy as we're modifying the list final List exitInfos = new ArrayList<>(applicationExitInfoList); + final @Nullable Long lastReportedTombstoneTimestamp = + AndroidEnvelopeCache.lastReportedTombstone(options); // search for the latest Tombstone to report it separately as we're gonna enrich it. The // latest @@ -152,8 +170,6 @@ public void run() { if (applicationExitInfo.getReason() == ApplicationExitInfo.REASON_CRASH_NATIVE) { latestTombstone = applicationExitInfo; // remove it, so it's not reported twice - // TODO: if we fail after this, we effectively lost the ApplicationExitInfo (maybe only - // remove after we reported it) exitInfos.remove(applicationExitInfo); break; } @@ -171,25 +187,151 @@ public void run() { if (latestTombstone.getTimestamp() < threshold) { options .getLogger() - .log(SentryLevel.DEBUG, "Latest Tombstones happened too long ago, returning early."); + .log(SentryLevel.DEBUG, "Latest Tombstone happened too long ago, returning early."); + return; + } + + if (lastReportedTombstoneTimestamp != null + && latestTombstone.getTimestamp() <= lastReportedTombstoneTimestamp) { + options + .getLogger() + .log(SentryLevel.DEBUG, "Latest Tombstone has already been reported, returning early."); return; } - reportAsSentryEvent(latestTombstone); + if (options.isReportHistoricalTombstones()) { + // report the remainder without enriching + reportNonEnrichedHistoricalTombstones(exitInfos, lastReportedTombstoneTimestamp); + } + + // report the latest Tombstone with enriching, if contexts are available, otherwise report it + // non-enriched + reportAsSentryEvent(latestTombstone, true); } @RequiresApi(api = Build.VERSION_CODES.R) - private void reportAsSentryEvent(ApplicationExitInfo exitInfo) { + private void reportNonEnrichedHistoricalTombstones( + List exitInfos, @Nullable Long lastReportedTombstoneTimestamp) { + // we reverse the list, because the OS puts errors in order of appearance, last-to-first + // and we want to write a marker file after each ANR has been processed, so in case the app + // gets killed meanwhile, we can proceed from the last reported ANR and not process the entire + // list again + Collections.reverse(exitInfos); + for (ApplicationExitInfo applicationExitInfo : exitInfos) { + if (applicationExitInfo.getReason() == ApplicationExitInfo.REASON_CRASH_NATIVE) { + if (applicationExitInfo.getTimestamp() < threshold) { + options + .getLogger() + .log(SentryLevel.DEBUG, "Tombstone happened too long ago %s.", applicationExitInfo); + continue; + } + + if (lastReportedTombstoneTimestamp != null + && applicationExitInfo.getTimestamp() <= lastReportedTombstoneTimestamp) { + options + .getLogger() + .log( + SentryLevel.DEBUG, + "Tombstone has already been reported %s.", + applicationExitInfo); + continue; + } + + reportAsSentryEvent(applicationExitInfo, false); // do not enrich past events + } + } + } + + @RequiresApi(api = Build.VERSION_CODES.R) + private void reportAsSentryEvent(ApplicationExitInfo exitInfo, boolean enrich) { SentryEvent event; try { - TombstoneParser parser = new TombstoneParser(exitInfo.getTraceInputStream()); + InputStream tombstoneInputStream = exitInfo.getTraceInputStream(); + if (tombstoneInputStream == null) { + logTombstoneFailure(exitInfo); + return; + } + + final TombstoneParser parser = new TombstoneParser(tombstoneInputStream); event = parser.parse(); - event.setTimestamp(DateUtils.getDateTime(exitInfo.getTimestamp())); } catch (IOException e) { - throw new RuntimeException(e); + logTombstoneFailure(exitInfo); + return; + } + + if (event == null) { + logTombstoneFailure(exitInfo); + return; } - scopes.captureEvent(event); + final long tombstoneTimestamp = exitInfo.getTimestamp(); + event.setTimestamp(DateUtils.getDateTime(tombstoneTimestamp)); + + final TombstoneHint tombstoneHint = + new TombstoneHint( + options.getFlushTimeoutMillis(), options.getLogger(), tombstoneTimestamp, enrich); + final Hint hint = HintUtils.createWithTypeCheckHint(tombstoneHint); + + final @NotNull SentryId sentryId = scopes.captureEvent(event, hint); + final boolean isEventDropped = sentryId.equals(SentryId.EMPTY_ID); + if (!isEventDropped) { + // Block until the event is flushed to disk and the last_reported_tombstone marker is + // updated + if (!tombstoneHint.waitFlush()) { + options + .getLogger() + .log( + SentryLevel.WARNING, + "Timed out waiting to flush Tombstone event to disk. Event: %s", + event.getEventId()); + } + } + } + + @RequiresApi(api = Build.VERSION_CODES.R) + private void logTombstoneFailure(ApplicationExitInfo exitInfo) { + options + .getLogger() + .log( + SentryLevel.WARNING, + "Native crash report from %s does not contain a valid tombstone.", + DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(exitInfo.getTimestamp()))); + } + } + + @ApiStatus.Internal + public static final class TombstoneHint extends BlockingFlushHint + implements Backfillable, NativeCrashExit { + + private final long tombstoneTimestamp; + private final boolean shouldEnrich; + + public TombstoneHint( + long flushTimeoutMillis, + @NotNull ILogger logger, + long tombstoneTimestamp, + boolean shouldEnrich) { + super(flushTimeoutMillis, logger); + this.tombstoneTimestamp = tombstoneTimestamp; + this.shouldEnrich = shouldEnrich; } + + @Override + public Long timestamp() { + return tombstoneTimestamp; + } + + @Override + public boolean shouldEnrich() { + return shouldEnrich; + } + + @Override + public boolean isFlushable(@Nullable SentryId eventId) { + return true; + } + + @Override + public void setFlushable(@NotNull SentryId eventId) {} } } diff --git a/sentry/src/main/java/io/sentry/hints/NativeCrashExit.java b/sentry/src/main/java/io/sentry/hints/NativeCrashExit.java new file mode 100644 index 00000000000..6f063c43dfc --- /dev/null +++ b/sentry/src/main/java/io/sentry/hints/NativeCrashExit.java @@ -0,0 +1,15 @@ +package io.sentry.hints; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Nullable; + +/** + * This is not sensationally useful right now. It only exists as marker interface to distinguish Tombstone events from AbnormalExits, which + * they are not. The timestamp is used to record the timestamp of the last reported native crash we retrieved from the ApplicationExitInfo. + */ +@ApiStatus.Internal +public interface NativeCrashExit { + /** When exactly the crash exit happened */ + @Nullable + Long timestamp(); +} From 6cbfdf8744c9d171f59d49104cc6dffcd9148eb3 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 17:29:52 +0100 Subject: [PATCH 12/35] sprinkle with TODOs to highlight next PR steps --- .../sentry/android/core/AnrV2EventProcessor.java | 16 ++++++++++++++++ .../android/core/TombstoneIntegration.java | 2 ++ .../main/java/io/sentry/cache/EnvelopeCache.java | 3 +++ 3 files changed, 21 insertions(+) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2EventProcessor.java b/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2EventProcessor.java index 91bae9d7fe7..00c556cea1a 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2EventProcessor.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2EventProcessor.java @@ -122,6 +122,12 @@ public AnrV2EventProcessor( return event; } + // TODO: right now, any event with a Backfillable Hint will get here. This is fine if the + // enrichment code is generically applicable to any Backfilling Event. However some parts + // in the ANRv2EventProcessor are specific to ANRs and currently override the tombstone + // events. Similar to the Integration we should find an abstraction split that allows to + // separate the generic from the specific. + // we always set exception values, platform, os and device even if the ANR is not enrich-able // even though the OS context may change in the meantime (OS update), we consider this an // edge-case @@ -255,6 +261,7 @@ private void setFingerprints(final @NotNull SentryEvent event, final @NotNull Ob // sentry does not yet have a capability to provide default server-side fingerprint rules, // so we're doing this on the SDK side to group background and foreground ANRs separately // even if they have similar stacktraces + // TODO: this will always set the fingerprint of tombstones to foreground-anr final boolean isBackgroundAnr = isBackgroundAnr(hint); if (event.getFingerprints() == null) { event.setFingerprints( @@ -385,6 +392,8 @@ private void setApp(final @NotNull SentryBaseEvent event, final @NotNull Object // TODO: not entirely correct, because we define background ANRs as not the ones of // IMPORTANCE_FOREGROUND, but this doesn't mean the app was in foreground when an ANR happened // but it's our best effort for now. We could serialize AppState in theory. + + // TODO: this will always be true of tombstones app.setInForeground(!isBackgroundAnr(hint)); final PackageInfo packageInfo = ContextUtils.getPackageInfo(context, buildInfoProvider); @@ -533,6 +542,9 @@ private void setStaticValues(final @NotNull SentryEvent event) { private void setPlatform(final @NotNull SentryBaseEvent event) { if (event.getPlatform() == null) { // this actually means JVM related. + // TODO: since we write this from the tombstone parser we are current unaffected. It is good + // that it doesn't overwrite previous platform values, however we still rely on the + // order in which each event processor was called. event.setPlatform(SentryBaseEvent.DEFAULT_PLATFORM); } } @@ -564,12 +576,16 @@ private void setExceptions(final @NotNull SentryEvent event, final @NotNull Obje // and make an exception out of its stacktrace final Mechanism mechanism = new Mechanism(); if (!((Backfillable) hint).shouldEnrich()) { + // TODO: this currently overrides the signalhandler mechanism we set in the TombstoneParser + // with a new type (can be the right choice down the road, but currently is + // unintentional, and it might be better to leave it close to the Native SDK) // we only enrich the latest ANR in the list, so this is historical mechanism.setType("HistoricalAppExitInfo"); } else { mechanism.setType("AppExitInfo"); } + // TODO: this currently overrides the tombstone exceptions final boolean isBackgroundAnr = isBackgroundAnr(hint); String message = "ANR"; if (isBackgroundAnr) { diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java index 225f7f3df82..6b53cef9cb4 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -40,6 +40,8 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +// TODO: This is very very close to ANRv2Integration in terms of mechanism. Find an abstraction +// split that separates the equivalent mechanism from the differing policy between the two. @ApiStatus.Internal public class TombstoneIntegration implements Integration, Closeable { static final long NINETY_DAYS_THRESHOLD = TimeUnit.DAYS.toMillis(91); diff --git a/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java b/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java index 893bef4ab90..ad35be2330f 100644 --- a/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java +++ b/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java @@ -122,6 +122,7 @@ private boolean storeInternal(final @NotNull SentryEnvelope envelope, final @Not if (HintUtils.hasType(hint, AbnormalExit.class)) { tryEndPreviousSession(hint); } + // TODO: adapt tryEndPreviousSession(hint); to CrashExit.class if (HintUtils.hasType(hint, SessionStart.class)) { movePreviousSession(currentSessionFile, previousSessionFile); @@ -129,6 +130,7 @@ private boolean storeInternal(final @NotNull SentryEnvelope envelope, final @Not boolean crashedLastRun = false; final File crashMarkerFile = new File(options.getCacheDirPath(), NATIVE_CRASH_MARKER_FILE); + // TODO: this should probably check whether the Native SDK integration is currently enabled or remove the marker file if it isn't. Otherwise, application that disable the Native SDK, will report a crash for the last run forever. if (crashMarkerFile.exists()) { crashedLastRun = true; } @@ -153,6 +155,7 @@ private boolean storeInternal(final @NotNull SentryEnvelope envelope, final @Not } } + // TODO: maybe set crashLastRun for tombstone too? SentryCrashLastRunState.getInstance().setCrashedLastRun(crashedLastRun); flushPreviousSession(); From 4b1919e9bcf3f3015639ac33bffcd747264392c2 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 17:31:08 +0100 Subject: [PATCH 13/35] fix typo --- sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java b/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java index ff61bab87d2..f0fbd03762d 100644 --- a/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java +++ b/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java @@ -143,7 +143,7 @@ public static final class JsonKeys { public static final String FRAMES = "frames"; public static final String REGISTERS = "registers"; public static final String SNAPSHOT = "snapshot"; - public static final String INSTRUCTION_ADDRESS_ADJUSTMENT = "instruction_add_adjustment"; + public static final String INSTRUCTION_ADDRESS_ADJUSTMENT = "instruction_addr_adjustment"; } @Override From d931ddc733d61f5d063e3d940f4b8305d2436624 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 17:47:13 +0100 Subject: [PATCH 14/35] implement TombstoneParser as Closable and close the tombstone stream --- .../java/io/sentry/android/core/TombstoneIntegration.java | 5 +++-- .../android/core/internal/tombstone/TombstoneParser.java | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java index 6b53cef9cb4..4c21b30275d 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -254,8 +254,9 @@ private void reportAsSentryEvent(ApplicationExitInfo exitInfo, boolean enrich) { return; } - final TombstoneParser parser = new TombstoneParser(tombstoneInputStream); - event = parser.parse(); + try (final TombstoneParser parser = new TombstoneParser(tombstoneInputStream)) { + event = parser.parse(); + } } catch (IOException e) { logTombstoneFailure(exitInfo); return; diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index 5e435bf5971..30b8e6c3951 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -11,6 +11,7 @@ import io.sentry.protocol.SentryStackFrame; import io.sentry.protocol.SentryStackTrace; import io.sentry.protocol.SentryThread; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -19,7 +20,7 @@ import java.util.Locale; import java.util.Map; -public class TombstoneParser { +public class TombstoneParser implements Closeable { private final InputStream tombstoneStream; private final Map excTypeValueMap = new HashMap<>(); @@ -199,4 +200,9 @@ private DebugMeta createDebugMeta(TombstoneProtos.Tombstone tombstone) { return debugMeta; } + + @Override + public void close() throws IOException { + tombstoneStream.close(); + } } From 65bc76bc0caeb899cdac8d5843190d8a8d0972a5 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 17:48:21 +0100 Subject: [PATCH 15/35] tighten code with final and null annotations. --- .../android/core/TombstoneIntegration.java | 12 ++-- .../internal/tombstone/TombstoneParser.java | 59 ++++++++++--------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java index 4c21b30275d..3d5e10bd92b 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -213,7 +213,8 @@ public void run() { @RequiresApi(api = Build.VERSION_CODES.R) private void reportNonEnrichedHistoricalTombstones( - List exitInfos, @Nullable Long lastReportedTombstoneTimestamp) { + final @NotNull List exitInfos, + final @Nullable Long lastReportedTombstoneTimestamp) { // we reverse the list, because the OS puts errors in order of appearance, last-to-first // and we want to write a marker file after each ANR has been processed, so in case the app // gets killed meanwhile, we can proceed from the last reported ANR and not process the entire @@ -245,10 +246,11 @@ private void reportNonEnrichedHistoricalTombstones( } @RequiresApi(api = Build.VERSION_CODES.R) - private void reportAsSentryEvent(ApplicationExitInfo exitInfo, boolean enrich) { - SentryEvent event; + private void reportAsSentryEvent( + final @NotNull ApplicationExitInfo exitInfo, final boolean enrich) { + final SentryEvent event; try { - InputStream tombstoneInputStream = exitInfo.getTraceInputStream(); + final InputStream tombstoneInputStream = exitInfo.getTraceInputStream(); if (tombstoneInputStream == null) { logTombstoneFailure(exitInfo); return; @@ -292,7 +294,7 @@ private void reportAsSentryEvent(ApplicationExitInfo exitInfo, boolean enrich) { } @RequiresApi(api = Build.VERSION_CODES.R) - private void logTombstoneFailure(ApplicationExitInfo exitInfo) { + private void logTombstoneFailure(final @NotNull ApplicationExitInfo exitInfo) { options .getLogger() .log( diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index 30b8e6c3951..7562a304d4b 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -25,7 +25,7 @@ public class TombstoneParser implements Closeable { private final InputStream tombstoneStream; private final Map excTypeValueMap = new HashMap<>(); - public TombstoneParser(InputStream tombstoneStream) { + public TombstoneParser(@NonNull final InputStream tombstoneStream) { this.tombstoneStream = tombstoneStream; // keep the current signal type -> value mapping for compatibility @@ -37,10 +37,13 @@ public TombstoneParser(InputStream tombstoneStream) { excTypeValueMap.put("SIGSEGV", "Segfault"); } + @NonNull public SentryEvent parse() throws IOException { - TombstoneProtos.Tombstone tombstone = TombstoneProtos.Tombstone.parseFrom(tombstoneStream); + @NonNull + final TombstoneProtos.Tombstone tombstone = + TombstoneProtos.Tombstone.parseFrom(tombstoneStream); - SentryEvent event = new SentryEvent(); + final SentryEvent event = new SentryEvent(); event.setLevel(SentryLevel.FATAL); // must use the "native" platform because otherwise the stack-trace wouldn't be correctly parsed @@ -50,6 +53,7 @@ public SentryEvent parse() throws IOException { event.setDebugMeta(createDebugMeta(tombstone)); event.setExceptions(createException(tombstone)); assert event.getExceptions() != null; + assert event.getExceptions().size() == 1; event.setThreads(createThreads(tombstone, event.getExceptions().get(0))); return event; @@ -57,17 +61,17 @@ public SentryEvent parse() throws IOException { @NonNull private List createThreads( - TombstoneProtos.Tombstone tombstone, SentryException exc) { - List threads = new ArrayList<>(); + @NonNull final TombstoneProtos.Tombstone tombstone, @NonNull final SentryException exc) { + final List threads = new ArrayList<>(); for (Map.Entry threadEntry : tombstone.getThreadsMap().entrySet()) { - TombstoneProtos.Thread threadEntryValue = threadEntry.getValue(); + final TombstoneProtos.Thread threadEntryValue = threadEntry.getValue(); - SentryThread thread = new SentryThread(); + final SentryThread thread = new SentryThread(); thread.setId(Long.valueOf(threadEntry.getKey())); thread.setName(threadEntryValue.getName()); - SentryStackTrace stacktrace = createStackTrace(threadEntryValue); + final SentryStackTrace stacktrace = createStackTrace(threadEntryValue); thread.setStacktrace(stacktrace); if (tombstone.getTid() == threadEntryValue.getId()) { thread.setCrashed(true); @@ -82,18 +86,18 @@ private List createThreads( } @NonNull - private static SentryStackTrace createStackTrace(TombstoneProtos.Thread thread) { - List frames = new ArrayList<>(); + private static SentryStackTrace createStackTrace(@NonNull final TombstoneProtos.Thread thread) { + final List frames = new ArrayList<>(); for (TombstoneProtos.BacktraceFrame frame : thread.getCurrentBacktraceList()) { - SentryStackFrame stackFrame = new SentryStackFrame(); + final SentryStackFrame stackFrame = new SentryStackFrame(); stackFrame.setPackage(frame.getFileName()); stackFrame.setFunction(frame.getFunctionName()); stackFrame.setInstructionAddr(String.format("0x%x", frame.getPc())); frames.add(0, stackFrame); } - SentryStackTrace stacktrace = new SentryStackTrace(); + final SentryStackTrace stacktrace = new SentryStackTrace(); stacktrace.setFrames(frames); // `libunwindstack` used for tombstones already applies instruction address adjustment: @@ -101,7 +105,7 @@ private static SentryStackTrace createStackTrace(TombstoneProtos.Thread thread) // prevent "processing" from doing it again. stacktrace.setInstructionAddressAdjustment("none"); - Map registers = new HashMap<>(); + final Map registers = new HashMap<>(); for (TombstoneProtos.Register register : thread.getRegistersList()) { registers.put(register.getName(), String.format("0x%x", register.getU64())); } @@ -111,27 +115,28 @@ private static SentryStackTrace createStackTrace(TombstoneProtos.Thread thread) } @NonNull - private List createException(TombstoneProtos.Tombstone tombstone) { - SentryException exception = new SentryException(); + private List createException(@NonNull TombstoneProtos.Tombstone tombstone) { + final SentryException exception = new SentryException(); if (tombstone.hasSignalInfo()) { - TombstoneProtos.Signal signalInfo = tombstone.getSignalInfo(); + final TombstoneProtos.Signal signalInfo = tombstone.getSignalInfo(); exception.setType(signalInfo.getName()); exception.setValue(excTypeValueMap.get(signalInfo.getName())); exception.setMechanism(createMechanismFromSignalInfo(signalInfo)); } exception.setThreadId((long) tombstone.getTid()); - List exceptions = new ArrayList<>(1); + final List exceptions = new ArrayList<>(1); exceptions.add(exception); return exceptions; } @NonNull - private static Mechanism createMechanismFromSignalInfo(TombstoneProtos.Signal signalInfo) { + private static Mechanism createMechanismFromSignalInfo( + @NonNull final TombstoneProtos.Signal signalInfo) { - Mechanism mechanism = new Mechanism(); + final Mechanism mechanism = new Mechanism(); // this follows the current processing triggers strictly, changing any of these // alters grouping and name (long-term we might want to have a tombstone mechanism) // TODO: if we align this with ANRv2 this would be overwritten in a BackfillingEventProcessor as @@ -144,7 +149,7 @@ private static Mechanism createMechanismFromSignalInfo(TombstoneProtos.Signal si mechanism.setHandled(false); mechanism.setSynthetic(true); - Map meta = new HashMap<>(); + final Map meta = new HashMap<>(); meta.put("number", signalInfo.getNumber()); meta.put("name", signalInfo.getName()); meta.put("code", signalInfo.getCode()); @@ -155,9 +160,9 @@ private static Mechanism createMechanismFromSignalInfo(TombstoneProtos.Signal si } @NonNull - private Message constructMessage(TombstoneProtos.Tombstone tombstone) { - Message message = new Message(); - TombstoneProtos.Signal signalInfo = tombstone.getSignalInfo(); + private Message constructMessage(@NonNull final TombstoneProtos.Tombstone tombstone) { + final Message message = new Message(); + final TombstoneProtos.Signal signalInfo = tombstone.getSignalInfo(); // reproduce the message `debuggerd` would use to dump the stack trace in logcat message.setFormatted( @@ -174,8 +179,8 @@ private Message constructMessage(TombstoneProtos.Tombstone tombstone) { return message; } - private DebugMeta createDebugMeta(TombstoneProtos.Tombstone tombstone) { - List images = new ArrayList<>(); + private DebugMeta createDebugMeta(@NonNull final TombstoneProtos.Tombstone tombstone) { + final List images = new ArrayList<>(); for (TombstoneProtos.MemoryMapping module : tombstone.getMemoryMappingsList()) { // exclude anonymous and non-executable maps @@ -184,7 +189,7 @@ private DebugMeta createDebugMeta(TombstoneProtos.Tombstone tombstone) { || !module.getExecute()) { continue; } - DebugImage image = new DebugImage(); + final DebugImage image = new DebugImage(); image.setCodeId(module.getBuildId()); image.setCodeFile(module.getMappingName()); image.setDebugId(module.getBuildId()); @@ -195,7 +200,7 @@ private DebugMeta createDebugMeta(TombstoneProtos.Tombstone tombstone) { images.add(image); } - DebugMeta debugMeta = new DebugMeta(); + final DebugMeta debugMeta = new DebugMeta(); debugMeta.setImages(images); return debugMeta; From 25f4089990767d256d849fd65be564c853d67e67 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 2 Dec 2025 17:52:42 +0100 Subject: [PATCH 16/35] eliminate obsolete null check --- .../java/io/sentry/android/core/TombstoneIntegration.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java index 3d5e10bd92b..730746a8c25 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -264,11 +264,6 @@ private void reportAsSentryEvent( return; } - if (event == null) { - logTombstoneFailure(exitInfo); - return; - } - final long tombstoneTimestamp = exitInfo.getTimestamp(); event.setTimestamp(DateUtils.getDateTime(tombstoneTimestamp)); From 7db5895f15c4e8bfdb88067f2a9afdb62bb2b474 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Wed, 3 Dec 2025 09:58:54 +0100 Subject: [PATCH 17/35] Deduplicate ApplicationExitInfo handling for corresponding Integrations via HistoryDispatcher * also update api dumps and formatting for sentry changes --- .../api/sentry-android-core.api | 22 +- .../sentry/android/core/AnrV2Integration.java | 164 ++---------- .../ApplicationExitInfoHistoryDispatcher.java | 245 ++++++++++++++++++ .../android/core/SentryAndroidOptions.java | 2 + .../android/core/TombstoneIntegration.java | 189 +++----------- .../core/cache/AndroidEnvelopeCache.java | 112 +++++--- .../android/core/AnrV2IntegrationTest.kt | 4 +- sentry/api/sentry.api | 7 + .../java/io/sentry/cache/EnvelopeCache.java | 4 +- .../java/io/sentry/hints/NativeCrashExit.java | 9 +- .../io/sentry/protocol/SentryStackTrace.java | 5 +- 11 files changed, 412 insertions(+), 351 deletions(-) create mode 100644 sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoHistoryDispatcher.java diff --git a/sentry-android-core/api/sentry-android-core.api b/sentry-android-core/api/sentry-android-core.api index ad3d0fcd38e..5cf93ac1715 100644 --- a/sentry-android-core/api/sentry-android-core.api +++ b/sentry-android-core/api/sentry-android-core.api @@ -339,6 +339,7 @@ public final class io/sentry/android/core/SentryAndroidOptions : io/sentry/Sentr public fun isEnableSystemEventBreadcrumbs ()Z public fun isEnableSystemEventBreadcrumbsExtras ()Z public fun isReportHistoricalAnrs ()Z + public fun isReportHistoricalTombstones ()Z public fun isTombstoneEnabled ()Z public fun setAnrEnabled (Z)V public fun setAnrReportInDebug (Z)V @@ -368,6 +369,7 @@ public final class io/sentry/android/core/SentryAndroidOptions : io/sentry/Sentr public fun setNativeHandlerStrategy (Lio/sentry/android/core/NdkHandlerStrategy;)V public fun setNativeSdkName (Ljava/lang/String;)V public fun setReportHistoricalAnrs (Z)V + public fun setReportHistoricalTombstones (Z)V public fun setTombstoneEnabled (Z)V } @@ -467,9 +469,21 @@ public class io/sentry/android/core/TombstoneIntegration : io/sentry/Integration public fun register (Lio/sentry/IScopes;Lio/sentry/SentryOptions;)V } -public class io/sentry/android/core/TombstoneIntegration$TombstoneProcessor : java/lang/Runnable { - public fun (Landroid/content/Context;Lio/sentry/IScopes;Lio/sentry/android/core/SentryAndroidOptions;Lio/sentry/transport/ICurrentDateProvider;)V - public fun run ()V +public final class io/sentry/android/core/TombstoneIntegration$TombstoneHint : io/sentry/hints/BlockingFlushHint, io/sentry/hints/Backfillable, io/sentry/hints/NativeCrashExit { + public fun (JLio/sentry/ILogger;JZ)V + public fun isFlushable (Lio/sentry/protocol/SentryId;)Z + public fun setFlushable (Lio/sentry/protocol/SentryId;)V + public fun shouldEnrich ()Z + public fun timestamp ()Ljava/lang/Long; +} + +public class io/sentry/android/core/TombstoneIntegration$TombstonePolicy : io/sentry/android/core/ApplicationExitInfoHistoryDispatcher$ApplicationExitInfoPolicy { + public fun (Lio/sentry/android/core/SentryAndroidOptions;)V + public fun buildReport (Landroid/app/ApplicationExitInfo;Z)Lio/sentry/android/core/ApplicationExitInfoHistoryDispatcher$Report; + public fun getLabel ()Ljava/lang/String; + public fun getLastReportedTimestamp ()Ljava/lang/Long; + public fun getTargetReason ()I + public fun shouldReportHistorical ()Z } public final class io/sentry/android/core/UserInteractionIntegration : android/app/Application$ActivityLifecycleCallbacks, io/sentry/Integration, java/io/Closeable { @@ -498,7 +512,9 @@ public final class io/sentry/android/core/ViewHierarchyEventProcessor : io/sentr } public final class io/sentry/android/core/cache/AndroidEnvelopeCache : io/sentry/cache/EnvelopeCache { + public static final field LAST_ANR_MARKER_LABEL Ljava/lang/String; public static final field LAST_ANR_REPORT Ljava/lang/String; + public static final field LAST_TOMBSTONE_MARKER_LABEL Ljava/lang/String; public static final field LAST_TOMBSTONE_REPORT Ljava/lang/String; public fun (Lio/sentry/android/core/SentryAndroidOptions;)V public fun getDirectory ()Ljava/io/File; diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2Integration.java b/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2Integration.java index c6d47cadcb4..af3a942c8cc 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2Integration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2Integration.java @@ -18,8 +18,6 @@ import io.sentry.android.core.cache.AndroidEnvelopeCache; import io.sentry.android.core.internal.threaddump.Lines; import io.sentry.android.core.internal.threaddump.ThreadDumpParser; -import io.sentry.cache.EnvelopeCache; -import io.sentry.cache.IEnvelopeCache; import io.sentry.hints.AbnormalExit; import io.sentry.hints.Backfillable; import io.sentry.hints.BlockingFlushHint; @@ -39,10 +37,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.concurrent.TimeUnit; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -50,9 +45,6 @@ @SuppressLint("NewApi") // we check this in AnrIntegrationFactory public class AnrV2Integration implements Integration, Closeable { - // using 91 to avoid timezone change hassle, 90 days is how long Sentry keeps the events - static final long NINETY_DAYS_THRESHOLD = TimeUnit.DAYS.toMillis(91); - private final @NotNull Context context; private final @NotNull ICurrentDateProvider dateProvider; private @Nullable SentryAndroidOptions options; @@ -92,9 +84,11 @@ public void register(@NotNull IScopes scopes, @NotNull SentryOptions options) { try { options .getExecutorService() - .submit(new AnrProcessor(context, scopes, this.options, dateProvider)); + .submit( + new ApplicationExitInfoHistoryDispatcher( + context, scopes, this.options, dateProvider, new AnrV2Policy(this.options))); } catch (Throwable e) { - options.getLogger().log(SentryLevel.DEBUG, "Failed to start AnrProcessor.", e); + options.getLogger().log(SentryLevel.DEBUG, "Failed to start ANR processor.", e); } options.getLogger().log(SentryLevel.DEBUG, "AnrV2Integration installed."); addIntegrationToSdkVersion("AnrV2"); @@ -108,132 +102,37 @@ public void close() throws IOException { } } - static class AnrProcessor implements Runnable { + private static final class AnrV2Policy + implements ApplicationExitInfoHistoryDispatcher.ApplicationExitInfoPolicy { - private final @NotNull Context context; - private final @NotNull IScopes scopes; private final @NotNull SentryAndroidOptions options; - private final long threshold; - - AnrProcessor( - final @NotNull Context context, - final @NotNull IScopes scopes, - final @NotNull SentryAndroidOptions options, - final @NotNull ICurrentDateProvider dateProvider) { - this.context = context; - this.scopes = scopes; + + AnrV2Policy(final @NotNull SentryAndroidOptions options) { this.options = options; - this.threshold = dateProvider.getCurrentTimeMillis() - NINETY_DAYS_THRESHOLD; } - @SuppressLint("NewApi") // we check this in AnrIntegrationFactory @Override - public void run() { - final ActivityManager activityManager = - (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); - - final List applicationExitInfoList = - activityManager.getHistoricalProcessExitReasons(null, 0, 0); - if (applicationExitInfoList.size() == 0) { - options.getLogger().log(SentryLevel.DEBUG, "No records in historical exit reasons."); - return; - } - - final IEnvelopeCache cache = options.getEnvelopeDiskCache(); - if (cache instanceof EnvelopeCache) { - if (options.isEnableAutoSessionTracking() - && !((EnvelopeCache) cache).waitPreviousSessionFlush()) { - options - .getLogger() - .log( - SentryLevel.WARNING, - "Timed out waiting to flush previous session to its own file."); - - // if we timed out waiting here, we can already flush the latch, because the timeout is - // big - // enough to wait for it only once and we don't have to wait again in - // PreviousSessionFinalizer - ((EnvelopeCache) cache).flushPreviousSession(); - } - } - - // making a deep copy as we're modifying the list - final List exitInfos = new ArrayList<>(applicationExitInfoList); - final @Nullable Long lastReportedAnrTimestamp = AndroidEnvelopeCache.lastReportedAnr(options); - - // search for the latest ANR to report it separately as we're gonna enrich it. The latest - // ANR will be first in the list, as it's filled last-to-first in order of appearance - ApplicationExitInfo latestAnr = null; - for (ApplicationExitInfo applicationExitInfo : exitInfos) { - if (applicationExitInfo.getReason() == ApplicationExitInfo.REASON_ANR) { - latestAnr = applicationExitInfo; - // remove it, so it's not reported twice - exitInfos.remove(applicationExitInfo); - break; - } - } - - if (latestAnr == null) { - options - .getLogger() - .log(SentryLevel.DEBUG, "No ANRs have been found in the historical exit reasons list."); - return; - } - - if (latestAnr.getTimestamp() < threshold) { - options - .getLogger() - .log(SentryLevel.DEBUG, "Latest ANR happened too long ago, returning early."); - return; - } - - if (lastReportedAnrTimestamp != null - && latestAnr.getTimestamp() <= lastReportedAnrTimestamp) { - options - .getLogger() - .log(SentryLevel.DEBUG, "Latest ANR has already been reported, returning early."); - return; - } + public @NotNull String getLabel() { + return "ANR"; + } - if (options.isReportHistoricalAnrs()) { - // report the remainder without enriching - reportNonEnrichedHistoricalAnrs(exitInfos, lastReportedAnrTimestamp); - } + @Override + public int getTargetReason() { + return ApplicationExitInfo.REASON_ANR; + } - // report the latest ANR with enriching, if contexts are available, otherwise report it - // non-enriched - reportAsSentryEvent(latestAnr, true); + @Override + public boolean shouldReportHistorical() { + return options.isReportHistoricalAnrs(); } - private void reportNonEnrichedHistoricalAnrs( - final @NotNull List exitInfos, final @Nullable Long lastReportedAnr) { - // we reverse the list, because the OS puts errors in order of appearance, last-to-first - // and we want to write a marker file after each ANR has been processed, so in case the app - // gets killed meanwhile, we can proceed from the last reported ANR and not process the entire - // list again - Collections.reverse(exitInfos); - for (ApplicationExitInfo applicationExitInfo : exitInfos) { - if (applicationExitInfo.getReason() == ApplicationExitInfo.REASON_ANR) { - if (applicationExitInfo.getTimestamp() < threshold) { - options - .getLogger() - .log(SentryLevel.DEBUG, "ANR happened too long ago %s.", applicationExitInfo); - continue; - } - - if (lastReportedAnr != null && applicationExitInfo.getTimestamp() <= lastReportedAnr) { - options - .getLogger() - .log(SentryLevel.DEBUG, "ANR has already been reported %s.", applicationExitInfo); - continue; - } - - reportAsSentryEvent(applicationExitInfo, false); // do not enrich past events - } - } + @Override + public @Nullable Long getLastReportedTimestamp() { + return AndroidEnvelopeCache.lastReportedAnr(options); } - private void reportAsSentryEvent( + @Override + public @Nullable ApplicationExitInfoHistoryDispatcher.Report buildReport( final @NotNull ApplicationExitInfo exitInfo, final boolean shouldEnrich) { final long anrTimestamp = exitInfo.getTimestamp(); final boolean isBackground = @@ -247,7 +146,7 @@ private void reportAsSentryEvent( SentryLevel.WARNING, "Not reporting ANR event as there was no thread dump for the ANR %s", exitInfo.toString()); - return; + return null; } final AnrV2Hint anrHint = new AnrV2Hint( @@ -284,19 +183,7 @@ private void reportAsSentryEvent( } } - final @NotNull SentryId sentryId = scopes.captureEvent(event, hint); - final boolean isEventDropped = sentryId.equals(SentryId.EMPTY_ID); - if (!isEventDropped) { - // Block until the event is flushed to disk and the last_reported_anr marker is updated - if (!anrHint.waitFlush()) { - options - .getLogger() - .log( - SentryLevel.WARNING, - "Timed out waiting to flush ANR event to disk. Event: %s", - event.getEventId()); - } - } + return new ApplicationExitInfoHistoryDispatcher.Report(event, hint, anrHint); } private @NotNull ParseResult parseThreadDump( @@ -379,6 +266,7 @@ public boolean ignoreCurrentThread() { return false; } + @NotNull @Override public Long timestamp() { return timestamp; diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoHistoryDispatcher.java b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoHistoryDispatcher.java new file mode 100644 index 00000000000..53b19face62 --- /dev/null +++ b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoHistoryDispatcher.java @@ -0,0 +1,245 @@ +package io.sentry.android.core; + +import android.app.ActivityManager; +import android.app.ApplicationExitInfo; +import android.content.Context; +import android.os.Build; +import androidx.annotation.RequiresApi; +import io.sentry.Hint; +import io.sentry.IScopes; +import io.sentry.SentryEvent; +import io.sentry.SentryLevel; +import io.sentry.cache.EnvelopeCache; +import io.sentry.cache.IEnvelopeCache; +import io.sentry.hints.BlockingFlushHint; +import io.sentry.protocol.SentryId; +import io.sentry.transport.ICurrentDateProvider; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +@ApiStatus.Internal +final class ApplicationExitInfoHistoryDispatcher implements Runnable { + + // using 91 to avoid timezone change hassle, 90 days is how long Sentry keeps the events + static final long NINETY_DAYS_THRESHOLD = TimeUnit.DAYS.toMillis(91); + + private final @NotNull Context context; + private final @NotNull IScopes scopes; + private final @NotNull SentryAndroidOptions options; + private final @NotNull ApplicationExitInfoPolicy policy; + private final long threshold; + + ApplicationExitInfoHistoryDispatcher( + final @NotNull Context context, + final @NotNull IScopes scopes, + final @NotNull SentryAndroidOptions options, + final @NotNull ICurrentDateProvider dateProvider, + final @NotNull ApplicationExitInfoPolicy policy) { + this.context = ContextUtils.getApplicationContext(context); + this.scopes = scopes; + this.options = options; + this.policy = policy; + this.threshold = dateProvider.getCurrentTimeMillis() - NINETY_DAYS_THRESHOLD; + } + + @RequiresApi(api = Build.VERSION_CODES.R) + @Override + public void run() { + final ActivityManager activityManager = + (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); + + if (activityManager == null) { + options.getLogger().log(SentryLevel.ERROR, "Failed to retrieve ActivityManager."); + return; + } + + final List applicationExitInfoList = + activityManager.getHistoricalProcessExitReasons(null, 0, 0); + + if (applicationExitInfoList.isEmpty()) { + options.getLogger().log(SentryLevel.DEBUG, "No records in historical exit reasons."); + return; + } + + waitPreviousSessionFlush(); + + final List exitInfos = new ArrayList<>(applicationExitInfoList); + final @Nullable Long lastReportedTimestamp = policy.getLastReportedTimestamp(); + + final ApplicationExitInfo latest = removeLatest(exitInfos); + if (latest == null) { + options + .getLogger() + .log( + SentryLevel.DEBUG, + "No %ss have been found in the historical exit reasons list.", + policy.getLabel()); + return; + } + + if (latest.getTimestamp() < threshold) { + options + .getLogger() + .log( + SentryLevel.DEBUG, + "Latest %s happened too long ago, returning early.", + policy.getLabel()); + return; + } + + if (lastReportedTimestamp != null && latest.getTimestamp() <= lastReportedTimestamp) { + options + .getLogger() + .log( + SentryLevel.DEBUG, + "Latest %s has already been reported, returning early.", + policy.getLabel()); + return; + } + + if (policy.shouldReportHistorical()) { + reportHistorical(exitInfos, lastReportedTimestamp); + } + + report(latest, true); + } + + private void waitPreviousSessionFlush() { + final IEnvelopeCache cache = options.getEnvelopeDiskCache(); + if (cache instanceof EnvelopeCache) { + if (options.isEnableAutoSessionTracking() + && !((EnvelopeCache) cache).waitPreviousSessionFlush()) { + options + .getLogger() + .log( + SentryLevel.WARNING, + "Timed out waiting to flush previous session to its own file."); + + // if we timed out waiting here, we can already flush the latch, because the timeout is + // big enough to wait for it only once and we don't have to wait again in + // PreviousSessionFinalizer + ((EnvelopeCache) cache).flushPreviousSession(); + } + } + } + + @RequiresApi(api = Build.VERSION_CODES.R) + private @Nullable ApplicationExitInfo removeLatest( + final @NotNull List exitInfos) { + for (ApplicationExitInfo applicationExitInfo : exitInfos) { + if (applicationExitInfo.getReason() == policy.getTargetReason()) { + exitInfos.remove(applicationExitInfo); + return applicationExitInfo; + } + } + return null; + } + + @RequiresApi(api = Build.VERSION_CODES.R) + private void reportHistorical( + final @NotNull List exitInfos, + final @Nullable Long lastReportedTimestamp) { + Collections.reverse(exitInfos); + for (ApplicationExitInfo applicationExitInfo : exitInfos) { + if (applicationExitInfo.getReason() == policy.getTargetReason()) { + if (applicationExitInfo.getTimestamp() < threshold) { + options + .getLogger() + .log( + SentryLevel.DEBUG, + "%s happened too long ago %s.", + policy.getLabel(), + applicationExitInfo); + continue; + } + + if (lastReportedTimestamp != null + && applicationExitInfo.getTimestamp() <= lastReportedTimestamp) { + options + .getLogger() + .log( + SentryLevel.DEBUG, + "%s has already been reported %s.", + policy.getLabel(), + applicationExitInfo); + continue; + } + + report(applicationExitInfo, false); // do not enrich past events + } + } + } + + private void report(final @NotNull ApplicationExitInfo exitInfo, final boolean enrich) { + final @Nullable Report report = policy.buildReport(exitInfo, enrich); + + if (report == null) { + return; + } + + final @NotNull SentryId sentryId = scopes.captureEvent(report.getEvent(), report.getHint()); + final boolean isEventDropped = sentryId.equals(SentryId.EMPTY_ID); + if (!isEventDropped) { + final @Nullable BlockingFlushHint flushHint = report.getFlushHint(); + if (flushHint != null && !flushHint.waitFlush()) { + options + .getLogger() + .log( + SentryLevel.WARNING, + "Timed out waiting to flush %s event to disk. Event: %s", + policy.getLabel(), + report.getEvent().getEventId()); + } + } + } + + interface ApplicationExitInfoPolicy { + @NotNull + String getLabel(); + + int getTargetReason(); + + boolean shouldReportHistorical(); + + @Nullable + Long getLastReportedTimestamp(); + + @Nullable + Report buildReport(@NotNull ApplicationExitInfo exitInfo, boolean enrich); + } + + public static final class Report { + private final @NotNull SentryEvent event; + private final @NotNull Hint hint; + private final @Nullable BlockingFlushHint flushHint; + + Report( + final @NotNull SentryEvent event, + final @NotNull Hint hint, + final @Nullable BlockingFlushHint flushHint) { + this.event = event; + this.hint = hint; + this.flushHint = flushHint; + } + + @NotNull + public SentryEvent getEvent() { + return event; + } + + @NotNull + public Hint getHint() { + return hint; + } + + @Nullable + public BlockingFlushHint getFlushHint() { + return flushHint; + } + } +} diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java index c84c26a565a..6b315fef3ca 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroidOptions.java @@ -604,10 +604,12 @@ public void setReportHistoricalAnrs(final boolean reportHistoricalAnrs) { this.reportHistoricalAnrs = reportHistoricalAnrs; } + @ApiStatus.Internal public boolean isReportHistoricalTombstones() { return reportHistoricalTombstones; } + @ApiStatus.Internal public void setReportHistoricalTombstones(final boolean reportHistoricalTombstones) { this.reportHistoricalTombstones = reportHistoricalTombstones; } diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java index 730746a8c25..d869dbf7be9 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -2,7 +2,6 @@ import static io.sentry.util.IntegrationUtils.addIntegrationToSdkVersion; -import android.app.ActivityManager; import android.app.ApplicationExitInfo; import android.content.Context; import android.os.Build; @@ -15,10 +14,9 @@ import io.sentry.SentryEvent; import io.sentry.SentryLevel; import io.sentry.SentryOptions; +import io.sentry.android.core.ApplicationExitInfoHistoryDispatcher.ApplicationExitInfoPolicy; import io.sentry.android.core.cache.AndroidEnvelopeCache; import io.sentry.android.core.internal.tombstone.TombstoneParser; -import io.sentry.cache.EnvelopeCache; -import io.sentry.cache.IEnvelopeCache; import io.sentry.hints.Backfillable; import io.sentry.hints.BlockingFlushHint; import io.sentry.hints.NativeCrashExit; @@ -32,20 +30,12 @@ import java.io.InputStream; import java.time.Instant; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -// TODO: This is very very close to ANRv2Integration in terms of mechanism. Find an abstraction -// split that separates the equivalent mechanism from the differing policy between the two. @ApiStatus.Internal public class TombstoneIntegration implements Integration, Closeable { - static final long NINETY_DAYS_THRESHOLD = TimeUnit.DAYS.toMillis(91); - private final @NotNull Context context; private final @NotNull ICurrentDateProvider dateProvider; private @Nullable SentryAndroidOptions options; @@ -87,9 +77,15 @@ public void register(@NotNull IScopes scopes, @NotNull SentryOptions options) { try { options .getExecutorService() - .submit(new TombstoneProcessor(context, scopes, this.options, dateProvider)); + .submit( + new ApplicationExitInfoHistoryDispatcher( + context, + scopes, + this.options, + dateProvider, + new TombstonePolicy(this.options))); } catch (Throwable e) { - options.getLogger().log(SentryLevel.DEBUG, "Failed to start TombstoneProcessor.", e); + options.getLogger().log(SentryLevel.DEBUG, "Failed to start tombstone processor.", e); } options.getLogger().log(SentryLevel.DEBUG, "TombstoneIntegration installed."); addIntegrationToSdkVersion("Tombstone"); @@ -104,156 +100,45 @@ public void close() throws IOException { } @ApiStatus.Internal - public static class TombstoneProcessor implements Runnable { + public static class TombstonePolicy implements ApplicationExitInfoPolicy { - @NotNull private final Context context; - @NotNull private final IScopes scopes; - @NotNull private final SentryAndroidOptions options; - private final long threshold; + private final @NotNull SentryAndroidOptions options; - public TombstoneProcessor( - @NotNull Context context, - @NotNull IScopes scopes, - @NotNull SentryAndroidOptions options, - @NotNull ICurrentDateProvider dateProvider) { - this.context = context; - this.scopes = scopes; + public TombstonePolicy(final @NotNull SentryAndroidOptions options) { this.options = options; - - this.threshold = dateProvider.getCurrentTimeMillis() - NINETY_DAYS_THRESHOLD; } @Override - @RequiresApi(api = Build.VERSION_CODES.R) - public void run() { - final ActivityManager activityManager = - (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); - - if (activityManager == null) { - options.getLogger().log(SentryLevel.ERROR, "Failed to retrieve ActivityManager."); - return; - } - - final List applicationExitInfoList; - applicationExitInfoList = activityManager.getHistoricalProcessExitReasons(null, 0, 0); - - if (applicationExitInfoList.isEmpty()) { - options.getLogger().log(SentryLevel.DEBUG, "No records in historical exit reasons."); - return; - } - - final IEnvelopeCache cache = options.getEnvelopeDiskCache(); - if (cache instanceof EnvelopeCache) { - if (options.isEnableAutoSessionTracking() - && !((EnvelopeCache) cache).waitPreviousSessionFlush()) { - options - .getLogger() - .log( - SentryLevel.WARNING, - "Timed out waiting to flush previous session to its own file."); - - // if we timed out waiting here, we can already flush the latch, because the timeout is - // big enough to wait for it only once and we don't have to wait again in - // PreviousSessionFinalizer - ((EnvelopeCache) cache).flushPreviousSession(); - } - } - - // making a deep copy as we're modifying the list - final List exitInfos = new ArrayList<>(applicationExitInfoList); - final @Nullable Long lastReportedTombstoneTimestamp = - AndroidEnvelopeCache.lastReportedTombstone(options); - - // search for the latest Tombstone to report it separately as we're gonna enrich it. The - // latest - // Tombstone will be first in the list, as it's filled last-to-first in order of appearance - ApplicationExitInfo latestTombstone = null; - for (ApplicationExitInfo applicationExitInfo : exitInfos) { - if (applicationExitInfo.getReason() == ApplicationExitInfo.REASON_CRASH_NATIVE) { - latestTombstone = applicationExitInfo; - // remove it, so it's not reported twice - exitInfos.remove(applicationExitInfo); - break; - } - } - - if (latestTombstone == null) { - options - .getLogger() - .log( - SentryLevel.DEBUG, - "No Tombstones have been found in the historical exit reasons list."); - return; - } - - if (latestTombstone.getTimestamp() < threshold) { - options - .getLogger() - .log(SentryLevel.DEBUG, "Latest Tombstone happened too long ago, returning early."); - return; - } - - if (lastReportedTombstoneTimestamp != null - && latestTombstone.getTimestamp() <= lastReportedTombstoneTimestamp) { - options - .getLogger() - .log(SentryLevel.DEBUG, "Latest Tombstone has already been reported, returning early."); - return; - } - - if (options.isReportHistoricalTombstones()) { - // report the remainder without enriching - reportNonEnrichedHistoricalTombstones(exitInfos, lastReportedTombstoneTimestamp); - } - - // report the latest Tombstone with enriching, if contexts are available, otherwise report it - // non-enriched - reportAsSentryEvent(latestTombstone, true); + public @NotNull String getLabel() { + return "Tombstone"; } @RequiresApi(api = Build.VERSION_CODES.R) - private void reportNonEnrichedHistoricalTombstones( - final @NotNull List exitInfos, - final @Nullable Long lastReportedTombstoneTimestamp) { - // we reverse the list, because the OS puts errors in order of appearance, last-to-first - // and we want to write a marker file after each ANR has been processed, so in case the app - // gets killed meanwhile, we can proceed from the last reported ANR and not process the entire - // list again - Collections.reverse(exitInfos); - for (ApplicationExitInfo applicationExitInfo : exitInfos) { - if (applicationExitInfo.getReason() == ApplicationExitInfo.REASON_CRASH_NATIVE) { - if (applicationExitInfo.getTimestamp() < threshold) { - options - .getLogger() - .log(SentryLevel.DEBUG, "Tombstone happened too long ago %s.", applicationExitInfo); - continue; - } + @Override + public int getTargetReason() { + return ApplicationExitInfo.REASON_CRASH_NATIVE; + } - if (lastReportedTombstoneTimestamp != null - && applicationExitInfo.getTimestamp() <= lastReportedTombstoneTimestamp) { - options - .getLogger() - .log( - SentryLevel.DEBUG, - "Tombstone has already been reported %s.", - applicationExitInfo); - continue; - } + @Override + public boolean shouldReportHistorical() { + return options.isReportHistoricalTombstones(); + } - reportAsSentryEvent(applicationExitInfo, false); // do not enrich past events - } - } + @Override + public @Nullable Long getLastReportedTimestamp() { + return AndroidEnvelopeCache.lastReportedTombstone(options); } @RequiresApi(api = Build.VERSION_CODES.R) - private void reportAsSentryEvent( + @Override + public @Nullable ApplicationExitInfoHistoryDispatcher.Report buildReport( final @NotNull ApplicationExitInfo exitInfo, final boolean enrich) { final SentryEvent event; try { final InputStream tombstoneInputStream = exitInfo.getTraceInputStream(); if (tombstoneInputStream == null) { logTombstoneFailure(exitInfo); - return; + return null; } try (final TombstoneParser parser = new TombstoneParser(tombstoneInputStream)) { @@ -261,7 +146,7 @@ private void reportAsSentryEvent( } } catch (IOException e) { logTombstoneFailure(exitInfo); - return; + return null; } final long tombstoneTimestamp = exitInfo.getTimestamp(); @@ -272,20 +157,7 @@ private void reportAsSentryEvent( options.getFlushTimeoutMillis(), options.getLogger(), tombstoneTimestamp, enrich); final Hint hint = HintUtils.createWithTypeCheckHint(tombstoneHint); - final @NotNull SentryId sentryId = scopes.captureEvent(event, hint); - final boolean isEventDropped = sentryId.equals(SentryId.EMPTY_ID); - if (!isEventDropped) { - // Block until the event is flushed to disk and the last_reported_tombstone marker is - // updated - if (!tombstoneHint.waitFlush()) { - options - .getLogger() - .log( - SentryLevel.WARNING, - "Timed out waiting to flush Tombstone event to disk. Event: %s", - event.getEventId()); - } - } + return new ApplicationExitInfoHistoryDispatcher.Report(event, hint, tombstoneHint); } @RequiresApi(api = Build.VERSION_CODES.R) @@ -316,6 +188,7 @@ public TombstoneHint( this.shouldEnrich = shouldEnrich; } + @NotNull @Override public Long timestamp() { return tombstoneTimestamp; diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java b/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java index b8727dad0c6..ddbfa0cc2dc 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java @@ -23,6 +23,8 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -82,35 +84,9 @@ private boolean storeInternalAndroid(@NotNull SentryEnvelope envelope, @NotNull } } - HintUtils.runIfHasType( - hint, - AnrV2Integration.AnrV2Hint.class, - (anrHint) -> { - final @Nullable Long timestamp = anrHint.timestamp(); - options - .getLogger() - .log( - SentryLevel.DEBUG, - "Writing last reported ANR marker with timestamp %d", - timestamp); - - writeLastReportedAnrMarker(timestamp); - }); - - HintUtils.runIfHasType( - hint, - TombstoneIntegration.TombstoneHint.class, - (tombstoneHint) -> { - final @Nullable Long timestamp = tombstoneHint.timestamp(); - options - .getLogger() - .log( - SentryLevel.DEBUG, - "Writing last reported Tombstone marker with timestamp %d", - timestamp); - - writeLastReportedTombstoneMarker(timestamp); - }); + for (TimestampMarkerHandler handler : TIMESTAMP_MARKER_HANDLERS) { + handler.handle(this, hint, options); + } return didStore; } @@ -150,9 +126,9 @@ public static boolean hasStartupCrashMarker(final @NotNull SentryOptions options final File crashMarkerFile = new File(outboxPath, STARTUP_CRASH_MARKER_FILE); try { final boolean exists = - options.getRuntimeManager().runWithRelaxedPolicy(() -> crashMarkerFile.exists()); + options.getRuntimeManager().runWithRelaxedPolicy(crashMarkerFile::exists); if (exists) { - if (!options.getRuntimeManager().runWithRelaxedPolicy(() -> crashMarkerFile.delete())) { + if (!options.getRuntimeManager().runWithRelaxedPolicy(crashMarkerFile::delete)) { options .getLogger() .log( @@ -173,11 +149,11 @@ public static boolean hasStartupCrashMarker(final @NotNull SentryOptions options private static @Nullable Long lastReportedMarker( final @NotNull SentryOptions options, @NotNull String reportFilename, - @NotNull String markerCategory) { + @NotNull String markerLabel) { final String cacheDirPath = Objects.requireNonNull( options.getCacheDirPath(), - "Cache dir path should be set for getting " + markerCategory + "s reported"); + "Cache dir path should be set for getting " + markerLabel + "s reported"); final File lastMarker = new File(cacheDirPath, reportFilename); try { @@ -191,10 +167,10 @@ public static boolean hasStartupCrashMarker(final @NotNull SentryOptions options .getLogger() .log( DEBUG, - "Last " + markerCategory + " marker does not exist. %s.", + "Last " + markerLabel + " marker does not exist. %s.", lastMarker.getAbsolutePath()); } else { - options.getLogger().log(ERROR, "Error reading last " + markerCategory + " marker", e); + options.getLogger().log(ERROR, "Error reading last " + markerLabel + " marker", e); } } return null; @@ -226,18 +202,68 @@ private void writeLastReportedMarker( } public static @Nullable Long lastReportedAnr(final @NotNull SentryOptions options) { - return lastReportedMarker(options, LAST_ANR_REPORT, "ANR"); - } - - private void writeLastReportedAnrMarker(final @Nullable Long timestamp) { - this.writeLastReportedMarker(timestamp, LAST_ANR_REPORT, "ANR"); + return lastReportedMarker(options, LAST_ANR_REPORT, LAST_ANR_MARKER_LABEL); } public static @Nullable Long lastReportedTombstone(final @NotNull SentryOptions options) { - return lastReportedMarker(options, LAST_TOMBSTONE_REPORT, "Tombstone"); + return lastReportedMarker(options, LAST_TOMBSTONE_REPORT, LAST_TOMBSTONE_MARKER_LABEL); } - private void writeLastReportedTombstoneMarker(final @Nullable Long timestamp) { - this.writeLastReportedMarker(timestamp, LAST_TOMBSTONE_REPORT, "Tombstone"); + private static final class TimestampMarkerHandler { + interface TimestampExtractor { + @NotNull + Long extract(T value); + } + + private final @NotNull Class type; + private final @NotNull String label; + private final @NotNull String reportFilename; + private final @NotNull TimestampExtractor timestampProvider; + + TimestampMarkerHandler( + final @NotNull Class type, + final @NotNull String label, + final @NotNull String reportFilename, + final @NotNull TimestampExtractor timestampProvider) { + this.type = type; + this.label = label; + this.reportFilename = reportFilename; + this.timestampProvider = timestampProvider; + } + + void handle( + final @NotNull AndroidEnvelopeCache cache, + final @NotNull Hint hint, + final @NotNull SentryAndroidOptions options) { + HintUtils.runIfHasType( + hint, + type, + (typedHint) -> { + final @NotNull Long timestamp = timestampProvider.extract(typedHint); + options + .getLogger() + .log( + SentryLevel.DEBUG, + "Writing last reported %s marker with timestamp %d", + label, + timestamp); + cache.writeLastReportedMarker(timestamp, reportFilename, label); + }); + } } + + public static final String LAST_TOMBSTONE_MARKER_LABEL = "Tombstone"; + public static final String LAST_ANR_MARKER_LABEL = "ANR"; + private static final List> TIMESTAMP_MARKER_HANDLERS = + Arrays.asList( + new TimestampMarkerHandler<>( + AnrV2Integration.AnrV2Hint.class, + LAST_ANR_MARKER_LABEL, + LAST_ANR_REPORT, + AnrV2Integration.AnrV2Hint::timestamp), + new TimestampMarkerHandler<>( + TombstoneIntegration.TombstoneHint.class, + LAST_TOMBSTONE_MARKER_LABEL, + LAST_TOMBSTONE_REPORT, + TombstoneIntegration.TombstoneHint::timestamp)); } diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2IntegrationTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2IntegrationTest.kt index af2c208440d..5975aaa774a 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2IntegrationTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2IntegrationTest.kt @@ -242,7 +242,7 @@ class AnrV2IntegrationTest { fun `when latest ANR is older than 90 days, does not capture events`() { val oldTimestamp = System.currentTimeMillis() - - AnrV2Integration.NINETY_DAYS_THRESHOLD - + ApplicationExitInfoHistoryDispatcher.NINETY_DAYS_THRESHOLD - TimeUnit.DAYS.toMillis(2) val integration = fixture.getSut(tmpDir) fixture.addAppExitInfo(timestamp = oldTimestamp) @@ -510,7 +510,7 @@ class AnrV2IntegrationTest { thread { Thread.sleep(200L) val sessionHint = HintUtils.createWithTypeCheckHint(SessionStartHint()) - fixture.options.envelopeDiskCache.store( + fixture.options.envelopeDiskCache.storeEnvelope( SentryEnvelope(SentryId.EMPTY_ID, null, emptyList()), sessionHint, ) diff --git a/sentry/api/sentry.api b/sentry/api/sentry.api index bee8b9e343e..8833ac2825e 100644 --- a/sentry/api/sentry.api +++ b/sentry/api/sentry.api @@ -4845,6 +4845,10 @@ public abstract interface class io/sentry/hints/Flushable { public abstract fun waitFlush ()Z } +public abstract interface class io/sentry/hints/NativeCrashExit { + public abstract fun timestamp ()Ljava/lang/Long; +} + public abstract interface class io/sentry/hints/Resettable { public abstract fun reset ()V } @@ -6215,11 +6219,13 @@ public final class io/sentry/protocol/SentryStackTrace : io/sentry/JsonSerializa public fun ()V public fun (Ljava/util/List;)V public fun getFrames ()Ljava/util/List; + public fun getInstructionAddressAdjustment ()Ljava/lang/String; public fun getRegisters ()Ljava/util/Map; public fun getSnapshot ()Ljava/lang/Boolean; public fun getUnknown ()Ljava/util/Map; public fun serialize (Lio/sentry/ObjectWriter;Lio/sentry/ILogger;)V public fun setFrames (Ljava/util/List;)V + public fun setInstructionAddressAdjustment (Ljava/lang/String;)V public fun setRegisters (Ljava/util/Map;)V public fun setSnapshot (Ljava/lang/Boolean;)V public fun setUnknown (Ljava/util/Map;)V @@ -6233,6 +6239,7 @@ public final class io/sentry/protocol/SentryStackTrace$Deserializer : io/sentry/ public final class io/sentry/protocol/SentryStackTrace$JsonKeys { public static final field FRAMES Ljava/lang/String; + public static final field INSTRUCTION_ADDRESS_ADJUSTMENT Ljava/lang/String; public static final field REGISTERS Ljava/lang/String; public static final field SNAPSHOT Ljava/lang/String; public fun ()V diff --git a/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java b/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java index ad35be2330f..9cc1ca9cf63 100644 --- a/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java +++ b/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java @@ -130,7 +130,9 @@ private boolean storeInternal(final @NotNull SentryEnvelope envelope, final @Not boolean crashedLastRun = false; final File crashMarkerFile = new File(options.getCacheDirPath(), NATIVE_CRASH_MARKER_FILE); - // TODO: this should probably check whether the Native SDK integration is currently enabled or remove the marker file if it isn't. Otherwise, application that disable the Native SDK, will report a crash for the last run forever. + // TODO: this should probably check whether the Native SDK integration is currently enabled or + // remove the marker file if it isn't. Otherwise, application that disable the Native SDK, + // will report a crash for the last run forever. if (crashMarkerFile.exists()) { crashedLastRun = true; } diff --git a/sentry/src/main/java/io/sentry/hints/NativeCrashExit.java b/sentry/src/main/java/io/sentry/hints/NativeCrashExit.java index 6f063c43dfc..097d7ab553b 100644 --- a/sentry/src/main/java/io/sentry/hints/NativeCrashExit.java +++ b/sentry/src/main/java/io/sentry/hints/NativeCrashExit.java @@ -1,15 +1,16 @@ package io.sentry.hints; import org.jetbrains.annotations.ApiStatus; -import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.NotNull; /** - * This is not sensationally useful right now. It only exists as marker interface to distinguish Tombstone events from AbnormalExits, which - * they are not. The timestamp is used to record the timestamp of the last reported native crash we retrieved from the ApplicationExitInfo. + * This is not sensationally useful right now. It only exists as marker interface to distinguish + * Tombstone events from AbnormalExits, which they are not. The timestamp is used to record the + * timestamp of the last reported native crash we retrieved from the ApplicationExitInfo. */ @ApiStatus.Internal public interface NativeCrashExit { /** When exactly the crash exit happened */ - @Nullable + @NotNull Long timestamp(); } diff --git a/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java b/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java index f0fbd03762d..f04982a4f69 100644 --- a/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java +++ b/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java @@ -67,8 +67,9 @@ public final class SentryStackTrace implements JsonUnknown, JsonSerializable { private @Nullable Boolean snapshot; /** - * This value indicates if, and how, `instruction_addr` values in the stack frames need to be adjusted before they are symbolicated. - * TODO: should we make this an enum or is a string value fine? + * This value indicates if, and how, `instruction_addr` values in the stack frames need to be + * adjusted before they are symbolicated. TODO: should we make this an enum or is a string value + * fine? * * @see SentryStackFrame#getInstructionAddr() * @see SentryStackFrame#setInstructionAddr(String) From a8ea2304f07a6109e9981cf96d53dacd62002d5f Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Wed, 3 Dec 2025 10:25:25 +0100 Subject: [PATCH 18/35] update tombstone message construction --- .../internal/tombstone/TombstoneParser.java | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index 7562a304d4b..f851207c9a8 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -165,16 +165,24 @@ private Message constructMessage(@NonNull final TombstoneProtos.Tombstone tombst final TombstoneProtos.Signal signalInfo = tombstone.getSignalInfo(); // reproduce the message `debuggerd` would use to dump the stack trace in logcat - message.setFormatted( - String.format( - Locale.ROOT, - "Fatal signal %s (%d), %s (%d), pid = %d (%s)", - signalInfo.getName(), - signalInfo.getNumber(), - signalInfo.getCodeName(), - signalInfo.getCode(), - tombstone.getPid(), - String.join(" ", tombstone.getCommandLineList()))); + String command = String.join(" ", tombstone.getCommandLineList()); + if (tombstone.hasSignalInfo()) { + String abortMessage = tombstone.getAbortMessage(); + message.setFormatted( + String.format( + Locale.ROOT, + "%sFatal signal %s (%d), %s (%d), pid = %d (%s)", + abortMessage != null ? abortMessage + ": " : "", + signalInfo.getName(), + signalInfo.getNumber(), + signalInfo.getCodeName(), + signalInfo.getCode(), + tombstone.getPid(), + command)); + } else { + message.setFormatted( + String.format(Locale.ROOT, "Fatal exit pid = %d (%s)", tombstone.getPid(), command)); + } return message; } From ed81771a2d2c73fd204bbabae8888d0b1655df59 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Wed, 3 Dec 2025 10:51:18 +0100 Subject: [PATCH 19/35] fix abortMessage check --- .../sentry/android/core/internal/tombstone/TombstoneParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index f851207c9a8..3b8a6f01700 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -172,7 +172,7 @@ private Message constructMessage(@NonNull final TombstoneProtos.Tombstone tombst String.format( Locale.ROOT, "%sFatal signal %s (%d), %s (%d), pid = %d (%s)", - abortMessage != null ? abortMessage + ": " : "", + !abortMessage.isEmpty() ? abortMessage + ": " : "", signalInfo.getName(), signalInfo.getNumber(), signalInfo.getCodeName(), From 125b0c44071164b2dcd868e96b0173fa7fa4f814 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Wed, 3 Dec 2025 17:23:51 +0100 Subject: [PATCH 20/35] convert AnrV2EventProcessor to a more generic ApplicationExitInfoEventProcessor this handles all events with Backfillable hint, but adds an interface HintEnricher, to allow hint-specific enrichment (like for ANRs) before and after the generic backfilling happened. --- .../api/sentry-android-core.api | 14 +- .../core/AndroidOptionsInitializer.java | 6 +- ...=> ApplicationExitInfoEventProcessor.java} | 242 +++++++++++------- .../core/AndroidOptionsInitializerTest.kt | 5 +- ... ApplicationExitInfoEventProcessorTest.kt} | 54 +++- .../sentry/android/core/SentryAndroidTest.kt | 3 +- 6 files changed, 203 insertions(+), 121 deletions(-) rename sentry-android-core/src/main/java/io/sentry/android/core/{AnrV2EventProcessor.java => ApplicationExitInfoEventProcessor.java} (80%) rename sentry-android-core/src/test/java/io/sentry/android/core/{AnrV2EventProcessorTest.kt => ApplicationExitInfoEventProcessorTest.kt} (94%) diff --git a/sentry-android-core/api/sentry-android-core.api b/sentry-android-core/api/sentry-android-core.api index 5cf93ac1715..0f4a1ddb4df 100644 --- a/sentry-android-core/api/sentry-android-core.api +++ b/sentry-android-core/api/sentry-android-core.api @@ -129,13 +129,6 @@ public final class io/sentry/android/core/AnrIntegrationFactory { public static fun create (Landroid/content/Context;Lio/sentry/android/core/BuildInfoProvider;)Lio/sentry/Integration; } -public final class io/sentry/android/core/AnrV2EventProcessor : io/sentry/BackfillingEventProcessor { - public fun (Landroid/content/Context;Lio/sentry/android/core/SentryAndroidOptions;Lio/sentry/android/core/BuildInfoProvider;)V - public fun getOrder ()Ljava/lang/Long; - public fun process (Lio/sentry/SentryEvent;Lio/sentry/Hint;)Lio/sentry/SentryEvent; - public fun process (Lio/sentry/protocol/SentryTransaction;Lio/sentry/Hint;)Lio/sentry/protocol/SentryTransaction; -} - public class io/sentry/android/core/AnrV2Integration : io/sentry/Integration, java/io/Closeable { public fun (Landroid/content/Context;)V public fun close ()V @@ -191,6 +184,13 @@ public final class io/sentry/android/core/AppState$LifecycleObserver : androidx/ public fun onStop (Landroidx/lifecycle/LifecycleOwner;)V } +public final class io/sentry/android/core/ApplicationExitInfoEventProcessor : io/sentry/BackfillingEventProcessor { + public fun (Landroid/content/Context;Lio/sentry/android/core/SentryAndroidOptions;Lio/sentry/android/core/BuildInfoProvider;)V + public fun getOrder ()Ljava/lang/Long; + public fun process (Lio/sentry/SentryEvent;Lio/sentry/Hint;)Lio/sentry/SentryEvent; + public fun process (Lio/sentry/protocol/SentryTransaction;Lio/sentry/Hint;)Lio/sentry/protocol/SentryTransaction; +} + public final class io/sentry/android/core/BuildConfig { public static final field BUILD_TYPE Ljava/lang/String; public static final field DEBUG Z diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java index f0ce04d0880..de1616baa15 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java @@ -188,10 +188,8 @@ static void initializeIntegrationsAndProcessors( options.addEventProcessor(new PerformanceAndroidEventProcessor(options, activityFramesTracker)); options.addEventProcessor(new ScreenshotEventProcessor(options, buildInfoProvider)); options.addEventProcessor(new ViewHierarchyEventProcessor(options)); - options.addEventProcessor(new AnrV2EventProcessor(context, options, buildInfoProvider)); - if (buildInfoProvider.getSdkInfoVersion() >= Build.VERSION_CODES.S) { - options.addEventProcessor(new TombstoneEventProcessor(context, options, buildInfoProvider)); - } + options.addEventProcessor( + new ApplicationExitInfoEventProcessor(context, options, buildInfoProvider)); if (options.getTransportGate() instanceof NoOpTransportGate) { options.setTransportGate(new AndroidTransportGate(options)); } diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2EventProcessor.java b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java similarity index 80% rename from sentry-android-core/src/main/java/io/sentry/android/core/AnrV2EventProcessor.java rename to sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java index 00c556cea1a..da7e6e6db0e 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/AnrV2EventProcessor.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java @@ -69,12 +69,12 @@ import org.jetbrains.annotations.Nullable; /** - * AnrV2Integration processes events on a background thread, hence the event processors will also be - * invoked on the same background thread, so we can safely read data from disk synchronously. + * Processes cached ApplicationExitInfo events (ANRs, tombstones) on a background thread, so we can + * safely read data from disk synchronously. */ @ApiStatus.Internal @WorkerThread -public final class AnrV2EventProcessor implements BackfillingEventProcessor { +public final class ApplicationExitInfoEventProcessor implements BackfillingEventProcessor { private final @NotNull Context context; @@ -86,7 +86,12 @@ public final class AnrV2EventProcessor implements BackfillingEventProcessor { private final @Nullable PersistingScopeObserver persistingScopeObserver; - public AnrV2EventProcessor( + // Only ANRv2 events are currently enriched with hint-specific content. + // This can be extended to other hints like CrashNativeHint. + private final @NotNull List hintEnrichers = + Collections.singletonList(new AnrHintEnricher()); + + public ApplicationExitInfoEventProcessor( final @NotNull Context context, final @NotNull SentryAndroidOptions options, final @NotNull BuildInfoProvider buildInfoProvider) { @@ -101,6 +106,15 @@ public AnrV2EventProcessor( sentryExceptionFactory = new SentryExceptionFactory(sentryStackTraceFactory); } + private @Nullable HintEnricher findEnricher(final @NotNull Object hint) { + for (HintEnricher enricher : hintEnrichers) { + if (enricher.supports(hint)) { + return enricher; + } + } + return null; + } + @Override public @NotNull SentryTransaction process( @NotNull SentryTransaction transaction, @NotNull Hint hint) { @@ -121,22 +135,17 @@ public AnrV2EventProcessor( "The event is not Backfillable, but has been passed to BackfillingEventProcessor, skipping."); return event; } + final @NotNull Backfillable backfillable = (Backfillable) unwrappedHint; + final @Nullable HintEnricher hintEnricher = findEnricher(unwrappedHint); - // TODO: right now, any event with a Backfillable Hint will get here. This is fine if the - // enrichment code is generically applicable to any Backfilling Event. However some parts - // in the ANRv2EventProcessor are specific to ANRs and currently override the tombstone - // events. Similar to the Integration we should find an abstraction split that allows to - // separate the generic from the specific. + if (hintEnricher != null) { + hintEnricher.applyPreEnrichment(event, backfillable, unwrappedHint); + } - // we always set exception values, platform, os and device even if the ANR is not enrich-able - // even though the OS context may change in the meantime (OS update), we consider this an - // edge-case - setExceptions(event, unwrappedHint); - setPlatform(event); mergeOS(event); setDevice(event); - if (!((Backfillable) unwrappedHint).shouldEnrich()) { + if (!backfillable.shouldEnrich()) { options .getLogger() .log( @@ -145,17 +154,21 @@ public AnrV2EventProcessor( return event; } - backfillScope(event, unwrappedHint); + backfillScope(event); - backfillOptions(event, unwrappedHint); + backfillOptions(event); setStaticValues(event); + if (hintEnricher != null) { + hintEnricher.applyPostEnrichment(event, backfillable, unwrappedHint); + } + return event; } // region scope persisted values - private void backfillScope(final @NotNull SentryEvent event, final @NotNull Object hint) { + private void backfillScope(final @NotNull SentryEvent event) { setRequest(event); setUser(event); setScopeTags(event); @@ -163,7 +176,7 @@ private void backfillScope(final @NotNull SentryEvent event, final @NotNull Obje setExtras(event); setContexts(event); setTransaction(event); - setFingerprints(event, hint); + setFingerprints(event); setLevel(event); setTrace(event); setReplayId(event); @@ -199,8 +212,11 @@ private boolean sampleReplay(final @NotNull SentryEvent event) { private void setReplayId(final @NotNull SentryEvent event) { @Nullable String persistedReplayId = readFromDisk(options, REPLAY_FILENAME, String.class); - final @NotNull File replayFolder = - new File(options.getCacheDirPath(), "replay_" + persistedReplayId); + @Nullable String cacheDirPath = options.getCacheDirPath(); + if (cacheDirPath == null) { + return; + } + final @NotNull File replayFolder = new File(cacheDirPath, "replay_" + persistedReplayId); if (!replayFolder.exists()) { if (!sampleReplay(event)) { return; @@ -209,7 +225,7 @@ private void setReplayId(final @NotNull SentryEvent event) { // latest replay folder that was modified before the ANR event. persistedReplayId = null; long lastModified = Long.MIN_VALUE; - final File[] dirs = new File(options.getCacheDirPath()).listFiles(); + final File[] dirs = new File(cacheDirPath).listFiles(); if (dirs != null) { for (File dir : dirs) { if (dir.isDirectory() && dir.getName().startsWith("replay_")) { @@ -235,9 +251,7 @@ private void setReplayId(final @NotNull SentryEvent event) { private void setTrace(final @NotNull SentryEvent event) { final SpanContext spanContext = readFromDisk(options, TRACE_FILENAME, SpanContext.class); if (event.getContexts().getTrace() == null) { - if (spanContext != null - && spanContext.getSpanId() != null - && spanContext.getTraceId() != null) { + if (spanContext != null) { event.getContexts().setTrace(spanContext); } } @@ -251,22 +265,12 @@ private void setLevel(final @NotNull SentryEvent event) { } @SuppressWarnings("unchecked") - private void setFingerprints(final @NotNull SentryEvent event, final @NotNull Object hint) { + private void setFingerprints(final @NotNull SentryEvent event) { final List fingerprint = (List) readFromDisk(options, FINGERPRINT_FILENAME, List.class); if (event.getFingerprints() == null) { event.setFingerprints(fingerprint); } - - // sentry does not yet have a capability to provide default server-side fingerprint rules, - // so we're doing this on the SDK side to group background and foreground ANRs separately - // even if they have similar stacktraces - // TODO: this will always set the fingerprint of tombstones to foreground-anr - final boolean isBackgroundAnr = isBackgroundAnr(hint); - if (event.getFingerprints() == null) { - event.setFingerprints( - Arrays.asList("{{ default }}", isBackgroundAnr ? "background-anr" : "foreground-anr")); - } } private void setTransaction(final @NotNull SentryEvent event) { @@ -373,28 +377,22 @@ private void setRequest(final @NotNull SentryBaseEvent event) { // endregion // region options persisted values - private void backfillOptions(final @NotNull SentryEvent event, final @NotNull Object hint) { + private void backfillOptions(final @NotNull SentryEvent event) { setRelease(event); setEnvironment(event); setDist(event); setDebugMeta(event); setSdk(event); - setApp(event, hint); + setApp(event); setOptionsTags(event); } - private void setApp(final @NotNull SentryBaseEvent event, final @NotNull Object hint) { + private void setApp(final @NotNull SentryBaseEvent event) { App app = event.getContexts().getApp(); if (app == null) { app = new App(); } app.setAppName(ContextUtils.getApplicationName(context)); - // TODO: not entirely correct, because we define background ANRs as not the ones of - // IMPORTANCE_FOREGROUND, but this doesn't mean the app was in foreground when an ANR happened - // but it's our best effort for now. We could serialize AppState in theory. - - // TODO: this will always be true of tombstones - app.setInForeground(!isBackgroundAnr(hint)); final PackageInfo packageInfo = ContextUtils.getPackageInfo(context, buildInfoProvider); if (packageInfo != null) { @@ -539,29 +537,13 @@ private void setStaticValues(final @NotNull SentryEvent event) { setSideLoadedInfo(event); } - private void setPlatform(final @NotNull SentryBaseEvent event) { + private void setDefaultPlatform(final @NotNull SentryBaseEvent event) { if (event.getPlatform() == null) { // this actually means JVM related. - // TODO: since we write this from the tombstone parser we are current unaffected. It is good - // that it doesn't overwrite previous platform values, however we still rely on the - // order in which each event processor was called. event.setPlatform(SentryBaseEvent.DEFAULT_PLATFORM); } } - @Nullable - private SentryThread findMainThread(final @Nullable List threads) { - if (threads != null) { - for (SentryThread thread : threads) { - final String name = thread.getName(); - if (name != null && name.equals("main")) { - return thread; - } - } - } - return null; - } - // by default we assume that the ANR is foreground, unless abnormalMechanism is "anr_background" private boolean isBackgroundAnr(final @NotNull Object hint) { if (hint instanceof AbnormalExit) { @@ -571,40 +553,6 @@ private boolean isBackgroundAnr(final @NotNull Object hint) { return false; } - private void setExceptions(final @NotNull SentryEvent event, final @NotNull Object hint) { - // AnrV2 threads contain a thread dump from the OS, so we just search for the main thread dump - // and make an exception out of its stacktrace - final Mechanism mechanism = new Mechanism(); - if (!((Backfillable) hint).shouldEnrich()) { - // TODO: this currently overrides the signalhandler mechanism we set in the TombstoneParser - // with a new type (can be the right choice down the road, but currently is - // unintentional, and it might be better to leave it close to the Native SDK) - // we only enrich the latest ANR in the list, so this is historical - mechanism.setType("HistoricalAppExitInfo"); - } else { - mechanism.setType("AppExitInfo"); - } - - // TODO: this currently overrides the tombstone exceptions - final boolean isBackgroundAnr = isBackgroundAnr(hint); - String message = "ANR"; - if (isBackgroundAnr) { - message = "Background " + message; - } - final ApplicationNotResponding anr = - new ApplicationNotResponding(message, Thread.currentThread()); - - SentryThread mainThread = findMainThread(event.getThreads()); - if (mainThread == null) { - // if there's no main thread in the event threads, we just create a dummy thread so the - // exception is properly created as well, but without stacktrace - mainThread = new SentryThread(); - mainThread.setStacktrace(new SentryStackTrace()); - } - event.setExceptions( - sentryExceptionFactory.getSentryExceptionsFromThread(mainThread, mechanism, anr)); - } - private void mergeUser(final @NotNull SentryBaseEvent event) { @Nullable User user = event.getUser(); if (user == null) { @@ -715,6 +663,106 @@ private void mergeOS(final @NotNull SentryBaseEvent event) { } event.getContexts().put(osNameKey, currentOS); } + // endregion + } + + private interface HintEnricher { + boolean supports(@NotNull Object hint); + + void applyPreEnrichment( + @NotNull SentryEvent event, @NotNull Backfillable hint, @NotNull Object rawHint); + + void applyPostEnrichment( + @NotNull SentryEvent event, @NotNull Backfillable hint, @NotNull Object rawHint); + } + + private final class AnrHintEnricher implements HintEnricher { + + @Override + public boolean supports(@NotNull Object hint) { + // TODO: not sure about this. all tests are written with AbnormalExit, but enrichment changes + // are ANR-specific. I called it AnrHintEnricher because that is what it does, but it + // actually triggers on AbnormalExit. Let me know what makes most sense. + return hint instanceof AbnormalExit; + } + + @Override + public void applyPreEnrichment( + @NotNull SentryEvent event, @NotNull Backfillable hint, @NotNull Object rawHint) { + final boolean isBackgroundAnr = isBackgroundAnr(rawHint); + setDefaultPlatform(event); + setAnrExceptions(event, hint, isBackgroundAnr); + } + + @Override + public void applyPostEnrichment( + @NotNull SentryEvent event, @NotNull Backfillable hint, @NotNull Object rawHint) { + final boolean isBackgroundAnr = isBackgroundAnr(rawHint); + setAppForeground(event, !isBackgroundAnr); + setDefaultAnrFingerprint(event, isBackgroundAnr); + } + + private void setDefaultAnrFingerprint( + final @NotNull SentryEvent event, final boolean isBackgroundAnr) { + if (event.getFingerprints() == null) { + event.setFingerprints( + Arrays.asList("{{ default }}", isBackgroundAnr ? "background-anr" : "foreground-anr")); + } + } + + private void setAppForeground( + final @NotNull SentryBaseEvent event, final boolean inForeground) { + App app = event.getContexts().getApp(); + if (app == null) { + app = new App(); + event.getContexts().setApp(app); + } + if (app.getInForeground() == null) { + app.setInForeground(inForeground); + } + } + + @Nullable + private SentryThread findMainThread(final @Nullable List threads) { + if (threads != null) { + for (SentryThread thread : threads) { + final String name = thread.getName(); + if (name != null && name.equals("main")) { + return thread; + } + } + } + return null; + } + + private void setAnrExceptions( + final @NotNull SentryEvent event, + final @NotNull Backfillable hint, + final boolean isBackgroundAnr) { + if (event.getExceptions() != null) { + return; + } + final Mechanism mechanism = new Mechanism(); + if (!hint.shouldEnrich()) { + mechanism.setType("HistoricalAppExitInfo"); + } else { + mechanism.setType("AppExitInfo"); + } + + String message = "ANR"; + if (isBackgroundAnr) { + message = "Background " + message; + } + final ApplicationNotResponding anr = + new ApplicationNotResponding(message, Thread.currentThread()); + + SentryThread mainThread = findMainThread(event.getThreads()); + if (mainThread == null) { + mainThread = new SentryThread(); + mainThread.setStacktrace(new SentryStackTrace()); + } + event.setExceptions( + sentryExceptionFactory.getSentryExceptionsFromThread(mainThread, mechanism, anr)); + } } - // endregion } diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt index 290ae6dea9d..c45808e19a4 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt @@ -259,9 +259,10 @@ class AndroidOptionsInitializerTest { } @Test - fun `AnrV2EventProcessor added to processors list`() { + fun `ApplicationExitInfoProcessor added to processors list`() { fixture.initSut() - val actual = fixture.sentryOptions.eventProcessors.firstOrNull { it is AnrV2EventProcessor } + val actual = + fixture.sentryOptions.eventProcessors.firstOrNull { it is ApplicationExitInfoEventProcessor } assertNotNull(actual) } diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2EventProcessorTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/ApplicationExitInfoEventProcessorTest.kt similarity index 94% rename from sentry-android-core/src/test/java/io/sentry/android/core/AnrV2EventProcessorTest.kt rename to sentry-android-core/src/test/java/io/sentry/android/core/ApplicationExitInfoEventProcessorTest.kt index b80d2838c4e..66090fc815e 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2EventProcessorTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/ApplicationExitInfoEventProcessorTest.kt @@ -47,6 +47,7 @@ import io.sentry.protocol.OperatingSystem import io.sentry.protocol.Request import io.sentry.protocol.Response import io.sentry.protocol.SdkVersion +import io.sentry.protocol.SentryException import io.sentry.protocol.SentryId import io.sentry.protocol.SentryStackFrame import io.sentry.protocol.SentryStackTrace @@ -74,7 +75,7 @@ import org.robolectric.shadows.ShadowActivityManager import org.robolectric.shadows.ShadowBuild @RunWith(AndroidJUnit4::class) -class AnrV2EventProcessorTest { +class ApplicationExitInfoEventProcessorTest { @get:Rule val tmpDir = TemporaryFolder() class Fixture { @@ -93,7 +94,7 @@ class AnrV2EventProcessorTest { populateOptionsCache: Boolean = false, replayErrorSampleRate: Double? = null, isSendDefaultPii: Boolean = true, - ): AnrV2EventProcessor { + ): ApplicationExitInfoEventProcessor { options.cacheDirPath = dir.newFolder().absolutePath options.environment = "release" options.isSendDefaultPii = isSendDefaultPii @@ -150,7 +151,7 @@ class AnrV2EventProcessorTest { } } - return AnrV2EventProcessor(context, options, buildInfo) + return ApplicationExitInfoEventProcessor(context, options, buildInfo) } fun persistScope(filename: String, entity: T) { @@ -204,7 +205,7 @@ class AnrV2EventProcessorTest { @Test fun `when backfillable event is not enrichable, sets different mechanism`() { - val hint = HintUtils.createWithTypeCheckHint(BackfillableHint(shouldEnrich = false)) + val hint = HintUtils.createWithTypeCheckHint(AbnormalExitHint(shouldEnrich = false)) val processed = processEvent(hint) @@ -213,7 +214,7 @@ class AnrV2EventProcessorTest { @Test fun `when backfillable event is not enrichable, sets platform`() { - val hint = HintUtils.createWithTypeCheckHint(BackfillableHint(shouldEnrich = false)) + val hint = HintUtils.createWithTypeCheckHint(AbnormalExitHint(shouldEnrich = false)) val processed = processEvent(hint) @@ -277,7 +278,7 @@ class AnrV2EventProcessorTest { @Test fun `when backfillable event is enrichable, still sets static data`() { - val hint = HintUtils.createWithTypeCheckHint(BackfillableHint()) + val hint = HintUtils.createWithTypeCheckHint(AbnormalExitHint()) val processed = processEvent(hint) @@ -352,11 +353,20 @@ class AnrV2EventProcessorTest { assertEquals("io.sentry.android.core.test", processed.contexts.app!!.appName) assertEquals("1.2.0", processed.contexts.app!!.appVersion) assertEquals("232", processed.contexts.app!!.appBuild) - assertEquals(true, processed.contexts.app!!.inForeground) + assertNull(processed.contexts.app!!.inForeground) // tags assertEquals("tag", processed.tags!!["option"]) } + @Test + fun `when ANR event is enrichable, sets foreground flag`() { + val hint = HintUtils.createWithTypeCheckHint(AbnormalExitHint()) + + val processed = processEvent(hint, populateOptionsCache = true) + + assertEquals(true, processed.contexts.app!!.inForeground) + } + @Test fun `if release is in wrong format, does not crash and leaves app version and build empty`() { val hint = HintUtils.createWithTypeCheckHint(BackfillableHint()) @@ -595,6 +605,29 @@ class AnrV2EventProcessorTest { assertEquals(listOf("{{ default }}", "foreground-anr"), processedForeground.fingerprints) } + @Test + fun `tombstone hint does not override platform or exceptions`() { + val hint = + HintUtils.createWithTypeCheckHint( + TombstoneIntegration.TombstoneHint( + fixture.options.flushTimeoutMillis, + NoOpLogger.getInstance(), + 0, + true, + ) + ) + + val processed = + processEvent(hint, populateScopeCache = false, populateOptionsCache = false) { + platform = "native" + exceptions = listOf(SentryException().apply { type = "NativeCrash" }) + } + + assertEquals("native", processed.platform) + assertEquals("NativeCrash", processed.exceptions!!.first().type) + assertNull(processed.fingerprints) + } + @Test fun `sets replayId when replay folder exists`() { val hint = HintUtils.createWithTypeCheckHint(BackfillableHint()) @@ -691,14 +724,17 @@ class AnrV2EventProcessorTest { return processor.process(original, hint)!! } - internal class AbnormalExitHint(val mechanism: String? = null) : AbnormalExit, Backfillable { + internal class AbnormalExitHint( + val mechanism: String? = null, + private val shouldEnrich: Boolean = true, + ) : AbnormalExit, Backfillable { override fun mechanism(): String? = mechanism override fun ignoreCurrentThread(): Boolean = false override fun timestamp(): Long? = null - override fun shouldEnrich(): Boolean = true + override fun shouldEnrich(): Boolean = shouldEnrich } internal class BackfillableHint(private val shouldEnrich: Boolean = true) : Backfillable { diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt index e2d92239f19..a9ab18098f3 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt @@ -47,7 +47,6 @@ import io.sentry.cache.tape.QueueFile import io.sentry.protocol.Contexts import io.sentry.protocol.SentryId import io.sentry.test.applyTestOptions -import io.sentry.test.initForTest import io.sentry.transport.NoOpEnvelopeCache import io.sentry.util.StringUtils import java.io.ByteArrayOutputStream @@ -539,7 +538,7 @@ class SentryAndroidTest { } assertTrue(optionsRef.eventProcessors.any { it is DefaultAndroidEventProcessor }) - assertTrue(optionsRef.eventProcessors.any { it is AnrV2EventProcessor }) + assertTrue(optionsRef.eventProcessors.any { it is ApplicationExitInfoEventProcessor }) } private fun prefillScopeCache(options: SentryOptions, cacheDir: String) { From 7e29fbf451912705af6bffda4cf7c11284dc6d93 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Wed, 3 Dec 2025 17:47:49 +0100 Subject: [PATCH 21/35] reintroduce, update and correct old inline docs where they make sense. --- .../core/ApplicationExitInfoEventProcessor.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java index da7e6e6db0e..0f3b8c3a113 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java @@ -87,7 +87,7 @@ public final class ApplicationExitInfoEventProcessor implements BackfillingEvent private final @Nullable PersistingScopeObserver persistingScopeObserver; // Only ANRv2 events are currently enriched with hint-specific content. - // This can be extended to other hints like CrashNativeHint. + // This can be extended to other hints like NativeCrashExit. private final @NotNull List hintEnrichers = Collections.singletonList(new AnrHintEnricher()); @@ -142,6 +142,8 @@ public ApplicationExitInfoEventProcessor( hintEnricher.applyPreEnrichment(event, backfillable, unwrappedHint); } + // We always set os and device even if the ApplicationExitInfo event is not enrich-able. + // The OS context may change in the meantime (OS update); we consider this an edge-case. mergeOS(event); setDevice(event); @@ -690,6 +692,7 @@ public boolean supports(@NotNull Object hint) { public void applyPreEnrichment( @NotNull SentryEvent event, @NotNull Backfillable hint, @NotNull Object rawHint) { final boolean isBackgroundAnr = isBackgroundAnr(rawHint); + // we always set exception values and default platform even if the ANR is not enrich-able setDefaultPlatform(event); setAnrExceptions(event, hint, isBackgroundAnr); } @@ -704,6 +707,9 @@ public void applyPostEnrichment( private void setDefaultAnrFingerprint( final @NotNull SentryEvent event, final boolean isBackgroundAnr) { + // sentry does not yet have a capability to provide default server-side fingerprint rules, + // so we're doing this on the SDK side to group background and foreground ANRs separately + // even if they have similar stacktraces. if (event.getFingerprints() == null) { event.setFingerprints( Arrays.asList("{{ default }}", isBackgroundAnr ? "background-anr" : "foreground-anr")); @@ -717,6 +723,9 @@ private void setAppForeground( app = new App(); event.getContexts().setApp(app); } + // TODO: not entirely correct, because we define background ANRs as not the ones of + // IMPORTANCE_FOREGROUND, but this doesn't mean the app was in foreground when an ANR + // happened but it's our best effort for now. We could serialize AppState in theory. if (app.getInForeground() == null) { app.setInForeground(inForeground); } @@ -742,8 +751,11 @@ private void setAnrExceptions( if (event.getExceptions() != null) { return; } + // AnrV2 threads contain a thread dump from the OS, so we just search for the main thread dump + // and make an exception out of its stacktrace final Mechanism mechanism = new Mechanism(); if (!hint.shouldEnrich()) { + // we only enrich the latest ANR in the list, so this is historical mechanism.setType("HistoricalAppExitInfo"); } else { mechanism.setType("AppExitInfo"); @@ -758,6 +770,8 @@ private void setAnrExceptions( SentryThread mainThread = findMainThread(event.getThreads()); if (mainThread == null) { + // if there's no main thread in the event threads, we just create a dummy thread so the + // exception is properly created as well, but without stacktrace mainThread = new SentryThread(); mainThread.setStacktrace(new SentryStackTrace()); } From 605a840ffa74bc462044978887e046e34cac73b3 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Fri, 5 Dec 2025 08:34:08 +0100 Subject: [PATCH 22/35] remove obsolete TombstoneEventProcessor --- .../api/sentry-android-core.api | 4 --- .../android/core/TombstoneEventProcessor.java | 26 ------------------- 2 files changed, 30 deletions(-) delete mode 100644 sentry-android-core/src/main/java/io/sentry/android/core/TombstoneEventProcessor.java diff --git a/sentry-android-core/api/sentry-android-core.api b/sentry-android-core/api/sentry-android-core.api index 0f4a1ddb4df..0700e744e29 100644 --- a/sentry-android-core/api/sentry-android-core.api +++ b/sentry-android-core/api/sentry-android-core.api @@ -459,10 +459,6 @@ public final class io/sentry/android/core/SystemEventsBreadcrumbsIntegration : i public fun register (Lio/sentry/IScopes;Lio/sentry/SentryOptions;)V } -public class io/sentry/android/core/TombstoneEventProcessor : io/sentry/BackfillingEventProcessor { - public fun (Landroid/content/Context;Lio/sentry/android/core/SentryAndroidOptions;Lio/sentry/android/core/BuildInfoProvider;)V -} - public class io/sentry/android/core/TombstoneIntegration : io/sentry/Integration, java/io/Closeable { public fun (Landroid/content/Context;)V public fun close ()V diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneEventProcessor.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneEventProcessor.java deleted file mode 100644 index ef064e13705..00000000000 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneEventProcessor.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.sentry.android.core; - -import android.content.Context; -import androidx.annotation.WorkerThread; -import io.sentry.BackfillingEventProcessor; -import org.jetbrains.annotations.ApiStatus; -import org.jetbrains.annotations.NotNull; - -/** originating from the Native SDK. */ -@ApiStatus.Internal -@WorkerThread -public class TombstoneEventProcessor implements BackfillingEventProcessor { - - @NotNull private final Context context; - @NotNull private final SentryAndroidOptions options; - @NotNull private final BuildInfoProvider buildInfoProvider; - - public TombstoneEventProcessor( - @NotNull Context context, - @NotNull SentryAndroidOptions options, - @NotNull BuildInfoProvider buildInfoProvider) { - this.context = context; - this.options = options; - this.buildInfoProvider = buildInfoProvider; - } -} From 7f6dbc94bd23aa688bd6b968e7d58a678fc0a083 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Fri, 5 Dec 2025 08:46:11 +0100 Subject: [PATCH 23/35] clean up tombstone error handling --- .../android/core/TombstoneIntegration.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java index d869dbf7be9..6d1c56db5ee 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/TombstoneIntegration.java @@ -137,15 +137,27 @@ public boolean shouldReportHistorical() { try { final InputStream tombstoneInputStream = exitInfo.getTraceInputStream(); if (tombstoneInputStream == null) { - logTombstoneFailure(exitInfo); + options + .getLogger() + .log( + SentryLevel.WARNING, + "No tombstone InputStream available for ApplicationExitInfo from %s", + DateTimeFormatter.ISO_INSTANT.format( + Instant.ofEpochMilli(exitInfo.getTimestamp()))); return null; } try (final TombstoneParser parser = new TombstoneParser(tombstoneInputStream)) { event = parser.parse(); } - } catch (IOException e) { - logTombstoneFailure(exitInfo); + } catch (Throwable e) { + options + .getLogger() + .log( + SentryLevel.WARNING, + "Failed to parse tombstone from %s: %s", + DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(exitInfo.getTimestamp())), + e.getMessage()); return null; } @@ -159,16 +171,6 @@ public boolean shouldReportHistorical() { return new ApplicationExitInfoHistoryDispatcher.Report(event, hint, tombstoneHint); } - - @RequiresApi(api = Build.VERSION_CODES.R) - private void logTombstoneFailure(final @NotNull ApplicationExitInfo exitInfo) { - options - .getLogger() - .log( - SentryLevel.WARNING, - "Native crash report from %s does not contain a valid tombstone.", - DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(exitInfo.getTimestamp()))); - } } @ApiStatus.Internal From 58a342f2f8f08d5fa8bc1533511652666efe93e2 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Fri, 5 Dec 2025 09:01:10 +0100 Subject: [PATCH 24/35] convert ApplicationExitInfoHistoryDispatcher.removeLatest() to use an iterator instead of for-each --- .../android/core/ApplicationExitInfoHistoryDispatcher.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoHistoryDispatcher.java b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoHistoryDispatcher.java index 53b19face62..79c3f19c6e5 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoHistoryDispatcher.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoHistoryDispatcher.java @@ -16,6 +16,7 @@ import io.sentry.transport.ICurrentDateProvider; import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; import org.jetbrains.annotations.ApiStatus; @@ -131,9 +132,10 @@ private void waitPreviousSessionFlush() { @RequiresApi(api = Build.VERSION_CODES.R) private @Nullable ApplicationExitInfo removeLatest( final @NotNull List exitInfos) { - for (ApplicationExitInfo applicationExitInfo : exitInfos) { + for (Iterator it = exitInfos.iterator(); it.hasNext(); ) { + ApplicationExitInfo applicationExitInfo = it.next(); if (applicationExitInfo.getReason() == policy.getTargetReason()) { - exitInfos.remove(applicationExitInfo); + it.remove(); return applicationExitInfo; } } From e4b7d78178b46d38f1f075e9563d567dd904af28 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 9 Dec 2025 14:41:00 +0100 Subject: [PATCH 25/35] add TombstoneIntegrationTest --- .../android/core/TombstoneIntegrationTest.kt | 486 ++++++++++++++++++ 1 file changed, 486 insertions(+) create mode 100644 sentry-android-core/src/test/java/io/sentry/android/core/TombstoneIntegrationTest.kt diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/TombstoneIntegrationTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/TombstoneIntegrationTest.kt new file mode 100644 index 00000000000..8c2f47192ac --- /dev/null +++ b/sentry-android-core/src/test/java/io/sentry/android/core/TombstoneIntegrationTest.kt @@ -0,0 +1,486 @@ +package io.sentry.android.core + +import android.app.ActivityManager +import android.app.ApplicationExitInfo +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.sentry.Hint +import io.sentry.ILogger +import io.sentry.IScopes +import io.sentry.SentryEnvelope +import io.sentry.SentryLevel +import io.sentry.android.core.TombstoneIntegration.TombstoneHint +import io.sentry.android.core.cache.AndroidEnvelopeCache +import io.sentry.cache.EnvelopeCache +import io.sentry.hints.DiskFlushNotification +import io.sentry.hints.SessionStartHint +import io.sentry.protocol.SentryId +import io.sentry.test.ImmediateExecutorService +import io.sentry.util.HintUtils +import java.io.File +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.mockito.kotlin.any +import org.mockito.kotlin.anyOrNull +import org.mockito.kotlin.argThat +import org.mockito.kotlin.atMost +import org.mockito.kotlin.check +import org.mockito.kotlin.inOrder +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.spy +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.robolectric.annotation.Config +import org.robolectric.shadow.api.Shadow +import org.robolectric.shadows.ShadowActivityManager +import org.robolectric.shadows.ShadowActivityManager.ApplicationExitInfoBuilder + +@RunWith(AndroidJUnit4::class) +@Config(sdk = [30]) +class TombstoneIntegrationTest { + @get:Rule val tmpDir = TemporaryFolder() + + class Fixture { + lateinit var context: Context + lateinit var shadowActivityManager: ShadowActivityManager + lateinit var lastReportedTombstoneFile: File + + val options = SentryAndroidOptions() + val scopes = mock() + val logger = mock() + + fun getSut( + dir: TemporaryFolder?, + useImmediateExecutorService: Boolean = true, + isTombstoneEnabled: Boolean = true, + flushTimeoutMillis: Long = 0L, + sessionFlushTimeoutMillis: Long = 0L, + lastReportedTombstoneTimestamp: Long? = null, + lastEventId: SentryId = SentryId(), + sessionTrackingEnabled: Boolean = true, + reportHistoricalTombstones: Boolean = true, + ): TombstoneIntegration { + options.run { + setLogger(this@Fixture.logger) + isDebug = true + cacheDirPath = dir?.newFolder()?.absolutePath + executorService = if (useImmediateExecutorService) ImmediateExecutorService() else mock() + this.isTombstoneEnabled = isTombstoneEnabled + this.flushTimeoutMillis = flushTimeoutMillis + this.sessionFlushTimeoutMillis = sessionFlushTimeoutMillis + this.isEnableAutoSessionTracking = sessionTrackingEnabled + this.isReportHistoricalTombstones = reportHistoricalTombstones + addInAppInclude("io.sentry.samples") + setEnvelopeDiskCache(EnvelopeCache.create(this)) + } + options.cacheDirPath?.let { cacheDir -> + lastReportedTombstoneFile = File(cacheDir, AndroidEnvelopeCache.LAST_TOMBSTONE_REPORT) + lastReportedTombstoneFile.writeText(lastReportedTombstoneTimestamp.toString()) + } + whenever(scopes.captureEvent(any(), anyOrNull())).thenReturn(lastEventId) + return TombstoneIntegration(context) + } + + fun addAppExitInfo( + reason: Int? = ApplicationExitInfo.REASON_CRASH_NATIVE, + timestamp: Long? = null, + importance: Int? = null, + addTrace: Boolean = true, + addBadTrace: Boolean = false, + ) { + val builder = ApplicationExitInfoBuilder.newBuilder() + if (reason != null) { + builder.setReason(reason) + } + if (timestamp != null) { + builder.setTimestamp(timestamp) + } + if (importance != null) { + builder.setImportance(importance) + } + val exitInfo = + spy(builder.build()) { + if (!addTrace) { + return + } + if (addBadTrace) { + whenever(mock.traceInputStream).thenReturn("XXXXX".byteInputStream()) + } else { + whenever(mock.traceInputStream) + .thenReturn(File("src/test/resources/tombstone.pb").inputStream()) + } + } + shadowActivityManager.addApplicationExitInfo(exitInfo) + } + } + + private val fixture = Fixture() + private val oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(10) + private val newTimestamp = oldTimestamp + TimeUnit.DAYS.toMillis(5) + + @BeforeTest + fun `set up`() { + fixture.context = ApplicationProvider.getApplicationContext() + val activityManager = + fixture.context.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager? + fixture.shadowActivityManager = Shadow.extract(activityManager) + } + + @Test + fun `when cacheDir is not set, does not process historical exits`() { + val integration = fixture.getSut(null, useImmediateExecutorService = false) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.options.executorService, never()).submit(any()) + } + + @Test + fun `when tombstone tracking is not enabled, does not process historical exits`() { + val integration = + fixture.getSut(tmpDir, isTombstoneEnabled = false, useImmediateExecutorService = false) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.options.executorService, never()).submit(any()) + } + + @Test + fun `when historical exit list is empty, does not process historical exits`() { + val integration = fixture.getSut(tmpDir) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when there are no tombstones in historical exits, does not capture events`() { + val integration = fixture.getSut(tmpDir) + fixture.addAppExitInfo(reason = null) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when latest Tombstone is older than 90 days, does not capture events`() { + val oldTimestamp = + System.currentTimeMillis() - + ApplicationExitInfoHistoryDispatcher.NINETY_DAYS_THRESHOLD - + TimeUnit.DAYS.toMillis(2) + val integration = fixture.getSut(tmpDir) + fixture.addAppExitInfo(timestamp = oldTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when latest Tombstone has already been reported, does not capture events`() { + val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = oldTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when no Tombstones have ever been reported, captures events`() { + val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = null) + fixture.addAppExitInfo(timestamp = oldTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when latest Tombstone has not been reported, captures event with enriching`() { + val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes) + .captureEvent( + check { + assertEquals(newTimestamp, it.timestamp.time) + assertEquals(SentryLevel.FATAL, it.level) + assertEquals("native", it.platform) + + val crashedThreadId = 21891.toLong() + assertEquals(crashedThreadId, it.exceptions!![0].threadId) + val crashedThread = it.threads!!.find { thread -> thread.id == crashedThreadId } + assertEquals("samples.android", crashedThread!!.name) + assertTrue(crashedThread.isCrashed!!) + + val image = + it.debugMeta?.images?.find { image -> + image.debugId == "f60b4b74005f33fb3ef3b98aa4546008" + } + assertNotNull(image) + assertEquals("/system/lib64/libcompiler_rt.so", image.codeFile) + assertEquals("0x764c32a000", image.imageAddr) + assertEquals(32768, image.imageSize) + }, + argThat { + val hint = HintUtils.getSentrySdkHint(this) + (hint as TombstoneHint).shouldEnrich() + }, + ) + } + + @Test + fun `waits for Tombstone events to be flushed on disk`() { + val integration = + fixture.getSut( + tmpDir, + lastReportedTombstoneTimestamp = oldTimestamp, + flushTimeoutMillis = 500L, + ) + fixture.addAppExitInfo(timestamp = newTimestamp) + + whenever(fixture.scopes.captureEvent(any(), any())).thenAnswer { invocation -> + val hint = HintUtils.getSentrySdkHint(invocation.getArgument(1)) as DiskFlushNotification + thread { + Thread.sleep(200L) + hint.markFlushed() + } + SentryId() + } + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes).captureEvent(any(), anyOrNull()) + // shouldn't fall into timed out state, because we marked event as flushed on another thread + verify(fixture.logger, never()) + .log( + any(), + argThat { startsWith("Timed out waiting to flush Tombstone event to disk.") }, + any(), + ) + } + + @Test + fun `when latest Tombstone event was dropped, does not block flushing`() { + val integration = + fixture.getSut( + tmpDir, + lastReportedTombstoneTimestamp = oldTimestamp, + lastEventId = SentryId.EMPTY_ID, + ) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes).captureEvent(any(), anyOrNull()) + // we do not call markFlushed, hence it should time out waiting for flush, but because + // we drop the event, it should not even come to this if-check + verify(fixture.logger, never()) + .log( + any(), + argThat { startsWith("Timed out waiting to flush Tombstone event to disk.") }, + any(), + ) + } + + @Test + fun `historical Tombstones are reported non-enriched`() { + val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp - 2 * 60 * 1000) + fixture.addAppExitInfo(timestamp = newTimestamp - 1 * 60 * 1000) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, times(2)) + .captureEvent( + any(), + argThat { + val hint = HintUtils.getSentrySdkHint(this) + !(hint as TombstoneHint).shouldEnrich() + }, + ) + } + + @Test + fun `when historical Tombstones flag is disabled, does not report`() { + val integration = + fixture.getSut( + tmpDir, + lastReportedTombstoneTimestamp = oldTimestamp, + reportHistoricalTombstones = false, + ) + fixture.addAppExitInfo(timestamp = newTimestamp - 2 * 60 * 1000) + fixture.addAppExitInfo(timestamp = newTimestamp - 1 * 60 * 1000) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + // only the latest tombstone is reported which should be enrichable + verify(fixture.scopes, atMost(1)) + .captureEvent( + any(), + argThat { + val hint = HintUtils.getSentrySdkHint(this) + (hint as TombstoneHint).shouldEnrich() + }, + ) + } + + @Test + fun `historical Tombstones are reported in reverse order to keep track of last reported Tombstone in a marker file`() { + val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) + // robolectric uses addFirst when adding exit infos, so the last one here will be the first on + // the list + fixture.addAppExitInfo(timestamp = newTimestamp - TimeUnit.DAYS.toMillis(2)) + fixture.addAppExitInfo(timestamp = newTimestamp - TimeUnit.DAYS.toMillis(1)) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + // the order is reverse here, so the oldest Tombstone will be reported first to keep track of + // last reported Tombstone in a marker file + inOrder(fixture.scopes) { + verify(fixture.scopes) + .captureEvent( + argThat { timestamp.time == newTimestamp - TimeUnit.DAYS.toMillis(2) }, + anyOrNull(), + ) + verify(fixture.scopes) + .captureEvent( + argThat { timestamp.time == newTimestamp - TimeUnit.DAYS.toMillis(1) }, + anyOrNull(), + ) + verify(fixture.scopes) + .captureEvent(argThat { timestamp.time == newTimestamp }, anyOrNull()) + } + } + + @Test + fun `Tombstone timestamp is passed with the hint`() { + val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes) + .captureEvent( + any(), + argThat { + val hint = HintUtils.getSentrySdkHint(this) + (hint as TombstoneHint).timestamp() == newTimestamp + }, + ) + } + + @Test + fun `awaits for previous session flush if cache is EnvelopeCache`() { + val integration = + fixture.getSut( + tmpDir, + lastReportedTombstoneTimestamp = oldTimestamp, + sessionFlushTimeoutMillis = 500L, + ) + fixture.addAppExitInfo(timestamp = newTimestamp) + + thread { + Thread.sleep(200L) + val sessionHint = HintUtils.createWithTypeCheckHint(SessionStartHint()) + fixture.options.envelopeDiskCache.storeEnvelope( + SentryEnvelope(SentryId.EMPTY_ID, null, emptyList()), + sessionHint, + ) + } + + integration.register(fixture.scopes, fixture.options) + + // we store envelope with StartSessionHint on different thread after some delay, which + // triggers the previous session flush, so no timeout + verify(fixture.logger, never()) + .log( + any(), + argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, + any(), + ) + } + + @Test + fun `does not await for previous session flush, if session tracking is disabled`() { + val integration = + fixture.getSut( + tmpDir, + lastReportedTombstoneTimestamp = oldTimestamp, + sessionFlushTimeoutMillis = 500L, + sessionTrackingEnabled = false, + ) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.logger, never()) + .log( + any(), + argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, + any(), + ) + verify(fixture.scopes).captureEvent(any(), any()) + } + + @Test + fun `flushes previous session latch, if timed out waiting`() { + val integration = + fixture.getSut( + tmpDir, + lastReportedTombstoneTimestamp = oldTimestamp, + sessionFlushTimeoutMillis = 500L, + ) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.logger) + .log( + any(), + argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, + any(), + ) + // should return true, because latch is 0 now + assertTrue((fixture.options.envelopeDiskCache as EnvelopeCache).waitPreviousSessionFlush()) + } + + @Test + fun `when traceInputStream is null, does not report Tombstone`() { + val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp, addTrace = false) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when traceInputStream has bad data, does not report Tombstone`() { + val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp, addBadTrace = true) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } +} From ace962f26c758b7261e4c6f5cac8f725ce947515 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 9 Dec 2025 15:31:21 +0100 Subject: [PATCH 26/35] replace assertions with explicit RuntimeException --- .../android/core/internal/tombstone/TombstoneParser.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index 3b8a6f01700..60f9376dfa6 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -52,8 +52,9 @@ public SentryEvent parse() throws IOException { event.setMessage(constructMessage(tombstone)); event.setDebugMeta(createDebugMeta(tombstone)); event.setExceptions(createException(tombstone)); - assert event.getExceptions() != null; - assert event.getExceptions().size() == 1; + if (event.getExceptions() == null || event.getExceptions().isEmpty()) { + throw new RuntimeException("Failed to decode exception information from tombstone"); + } event.setThreads(createThreads(tombstone, event.getExceptions().get(0))); return event; From 8cf561e1255293822c5de74d629eac415a51d85d Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 9 Dec 2025 15:43:37 +0100 Subject: [PATCH 27/35] use test base class for AEI --- .../android/core/AnrV2IntegrationTest.kt | 565 +++--------------- .../ApplicationExitIntegrationTestBase.kt | 435 ++++++++++++++ .../android/core/TombstoneIntegrationTest.kt | 518 ++-------------- 3 files changed, 576 insertions(+), 942 deletions(-) create mode 100644 sentry-android-core/src/test/java/io/sentry/android/core/ApplicationExitIntegrationTestBase.kt diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2IntegrationTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2IntegrationTest.kt index 5975aaa774a..a1a35facf56 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2IntegrationTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/AnrV2IntegrationTest.kt @@ -2,124 +2,56 @@ package io.sentry.android.core import android.app.ActivityManager import android.app.ApplicationExitInfo -import android.content.Context -import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 import io.sentry.Hint -import io.sentry.ILogger -import io.sentry.IScopes -import io.sentry.SentryEnvelope -import io.sentry.SentryLevel +import io.sentry.SentryEvent import io.sentry.android.core.AnrV2Integration.AnrV2Hint import io.sentry.android.core.cache.AndroidEnvelopeCache -import io.sentry.cache.EnvelopeCache -import io.sentry.hints.DiskFlushNotification -import io.sentry.hints.SessionStartHint -import io.sentry.protocol.SentryId -import io.sentry.test.ImmediateExecutorService import io.sentry.util.HintUtils -import java.io.File -import java.util.concurrent.TimeUnit -import kotlin.concurrent.thread -import kotlin.test.BeforeTest import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull -import kotlin.test.assertTrue -import org.junit.Rule -import org.junit.rules.TemporaryFolder import org.junit.runner.RunWith import org.mockito.kotlin.any -import org.mockito.kotlin.anyOrNull import org.mockito.kotlin.argThat -import org.mockito.kotlin.atMost import org.mockito.kotlin.check -import org.mockito.kotlin.inOrder -import org.mockito.kotlin.mock -import org.mockito.kotlin.never import org.mockito.kotlin.spy -import org.mockito.kotlin.times import org.mockito.kotlin.verify import org.mockito.kotlin.whenever import org.robolectric.annotation.Config -import org.robolectric.shadow.api.Shadow -import org.robolectric.shadows.ShadowActivityManager import org.robolectric.shadows.ShadowActivityManager.ApplicationExitInfoBuilder @RunWith(AndroidJUnit4::class) @Config(sdk = [30]) -class AnrV2IntegrationTest { - @get:Rule val tmpDir = TemporaryFolder() - - class Fixture { - lateinit var context: Context - lateinit var shadowActivityManager: ShadowActivityManager - lateinit var lastReportedAnrFile: File - - val options = SentryAndroidOptions() - val scopes = mock() - val logger = mock() - - fun getSut( - dir: TemporaryFolder?, - useImmediateExecutorService: Boolean = true, - isAnrEnabled: Boolean = true, - flushTimeoutMillis: Long = 0L, - sessionFlushTimeoutMillis: Long = 0L, - lastReportedAnrTimestamp: Long? = null, - lastEventId: SentryId = SentryId(), - sessionTrackingEnabled: Boolean = true, - reportHistoricalAnrs: Boolean = true, - attachAnrThreadDump: Boolean = false, - ): AnrV2Integration { - options.run { - setLogger(this@Fixture.logger) - isDebug = true - cacheDirPath = dir?.newFolder()?.absolutePath - executorService = if (useImmediateExecutorService) ImmediateExecutorService() else mock() - this.isAnrEnabled = isAnrEnabled - this.flushTimeoutMillis = flushTimeoutMillis - this.sessionFlushTimeoutMillis = sessionFlushTimeoutMillis - this.isEnableAutoSessionTracking = sessionTrackingEnabled - this.isReportHistoricalAnrs = reportHistoricalAnrs - this.isAttachAnrThreadDump = attachAnrThreadDump - addInAppInclude("io.sentry.samples") - setEnvelopeDiskCache(EnvelopeCache.create(this)) - } - options.cacheDirPath?.let { cacheDir -> - lastReportedAnrFile = File(cacheDir, AndroidEnvelopeCache.LAST_ANR_REPORT) - lastReportedAnrFile.writeText(lastReportedAnrTimestamp.toString()) - } - whenever(scopes.captureEvent(any(), anyOrNull())).thenReturn(lastEventId) - return AnrV2Integration(context) - } - - fun addAppExitInfo( - reason: Int? = ApplicationExitInfo.REASON_ANR, - timestamp: Long? = null, - importance: Int? = null, - addTrace: Boolean = true, - addBadTrace: Boolean = false, - ) { - val builder = ApplicationExitInfoBuilder.newBuilder() - if (reason != null) { - builder.setReason(reason) - } - if (timestamp != null) { - builder.setTimestamp(timestamp) - } - if (importance != null) { - builder.setImportance(importance) - } - val exitInfo = - spy(builder.build()) { - if (!addTrace) { - return - } - if (addBadTrace) { - whenever(mock.traceInputStream) - .thenReturn( - """ +class AnrV2IntegrationTest : ApplicationExitIntegrationTestBase() { + + override val config = + IntegrationTestConfig( + setEnabledFlag = { isAnrEnabled = it }, + setReportHistoricalFlag = { isReportHistoricalAnrs = it }, + createIntegration = { context -> AnrV2Integration(context) }, + lastReportedFileName = AndroidEnvelopeCache.LAST_ANR_REPORT, + defaultExitReason = ApplicationExitInfo.REASON_ANR, + hintAccessors = + HintAccessors( + cast = { it as AnrV2Hint }, + shouldEnrich = { it.shouldEnrich() }, + timestamp = { it.timestamp() }, + ), + addExitInfo = { reason, timestamp, importance, addTrace, addBadTrace -> + val builder = ApplicationExitInfoBuilder.newBuilder() + reason?.let { builder.setReason(it) } + timestamp?.let { builder.setTimestamp(it) } + importance?.let { builder.setImportance(it) } + val exitInfo = + spy(builder.build()) { + if (!addTrace) { + return@spy + } + if (addBadTrace) { + whenever(mock.traceInputStream) + .thenReturn( + """ Subject: Input dispatching timed out (7985007 com.example.app/com.example.app.ui.MainActivity (server) is not responding. Waited 5000ms for FocusEvent(hasFocus=false)) Here are no Binder-related exception messages available. Pid(12233) have D state thread(tid:12236 name:Signal Catcher) @@ -143,13 +75,13 @@ class AnrV2IntegrationTest { ----- Waiting Channels: pid 12233 at 2024-11-13 19:48:09.980104540+0530 ----- Cmd line: com.example.app:mainProcess """ - .trimIndent() - .byteInputStream() - ) - } else { - whenever(mock.traceInputStream) - .thenReturn( - """ + .trimIndent() + .byteInputStream() + ) + } else { + whenever(mock.traceInputStream) + .thenReturn( + """ "main" prio=5 tid=1 Blocked | group="main" sCount=1 ucsCount=0 flags=1 obj=0x72a985e0 self=0xb400007cabc57380 | sysTid=28941 nice=-10 cgrp=top-app sched=0/0 handle=0x7deceb74f8 @@ -179,154 +111,58 @@ class AnrV2IntegrationTest { native: #03 pc 00000000000530b8 /apex/com.android.runtime/lib64/bionic/libc.so (__start_thread+64) (BuildId: 01331f74b0bb2cb958bdc15282b8ec7b) (no managed stack frames) """ - .trimIndent() - .byteInputStream() - ) + .trimIndent() + .byteInputStream() + ) + } } - } - shadowActivityManager.addApplicationExitInfo(exitInfo) - } - } - - private val fixture = Fixture() - private val oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(10) - private val newTimestamp = oldTimestamp + TimeUnit.DAYS.toMillis(5) - - @BeforeTest - fun `set up`() { - fixture.context = ApplicationProvider.getApplicationContext() - val activityManager = - fixture.context.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager? - fixture.shadowActivityManager = Shadow.extract(activityManager) - } - - @Test - fun `when cacheDir is not set, does not process historical exits`() { - val integration = fixture.getSut(null, useImmediateExecutorService = false) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.options.executorService, never()).submit(any()) - } - - @Test - fun `when anr tracking is not enabled, does not process historical exits`() { - val integration = - fixture.getSut(tmpDir, isAnrEnabled = false, useImmediateExecutorService = false) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.options.executorService, never()).submit(any()) - } - - @Test - fun `when historical exit list is empty, does not process historical exits`() { - val integration = fixture.getSut(tmpDir) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when there are no ANRs in historical exits, does not capture events`() { - val integration = fixture.getSut(tmpDir) - fixture.addAppExitInfo(reason = null) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when latest ANR is older than 90 days, does not capture events`() { - val oldTimestamp = - System.currentTimeMillis() - - ApplicationExitInfoHistoryDispatcher.NINETY_DAYS_THRESHOLD - - TimeUnit.DAYS.toMillis(2) - val integration = fixture.getSut(tmpDir) - fixture.addAppExitInfo(timestamp = oldTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when latest ANR has already been reported, does not capture events`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = oldTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when no ANRs have ever been reported, captures events`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = null) - fixture.addAppExitInfo(timestamp = oldTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when latest ANR has not been reported, captures event with enriching`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes) - .captureEvent( - check { - assertEquals(newTimestamp, it.timestamp.time) - assertEquals(SentryLevel.FATAL, it.level) - val mainThread = it.threads!!.first() - assertEquals("main", mainThread.name) - assertEquals(1, mainThread.id) - assertEquals("Blocked", mainThread.state) - assertEquals(true, mainThread.isCrashed) - assertEquals(true, mainThread.isMain) - assertEquals("0x0d3a2f0a", mainThread.heldLocks!!.values.first().address) - assertEquals(5, mainThread.heldLocks!!.values.first().threadId) - val lastFrame = mainThread.stacktrace!!.frames!!.last() - assertEquals("io.sentry.samples.android.MainActivity$2", lastFrame.module) - assertEquals("MainActivity.java", lastFrame.filename) - assertEquals("run", lastFrame.function) - assertEquals(177, lastFrame.lineno) - assertEquals(true, lastFrame.isInApp) - val otherThread = it.threads!![1] - assertEquals("perfetto_hprof_listener", otherThread.name) - assertEquals(7, otherThread.id) - assertEquals("Native", otherThread.state) - assertEquals(false, otherThread.isCrashed) - assertEquals(false, otherThread.isMain) - val firstFrame = otherThread.stacktrace!!.frames!!.first() - assertEquals("/apex/com.android.runtime/lib64/bionic/libc.so", firstFrame.`package`) - assertEquals("__start_thread", firstFrame.function) - assertEquals(64, firstFrame.lineno) - assertEquals("0x00000000000530b8", firstFrame.instructionAddr) - assertEquals("native", firstFrame.platform) - assertEquals("rel:741f3301-bbb0-b92c-58bd-c15282b8ec7b", firstFrame.addrMode) + shadowActivityManager.addApplicationExitInfo(exitInfo) + }, + flushLogPrefix = "Timed out waiting to flush ANR event to disk.", + ) - val image = - it.debugMeta?.images?.find { it.debugId == "741f3301-bbb0-b92c-58bd-c15282b8ec7b" } - assertNotNull(image) - assertEquals("/apex/com.android.runtime/lib64/bionic/libc.so", image.codeFile) - }, - argThat { - val hint = HintUtils.getSentrySdkHint(this) - (hint as AnrV2Hint).shouldEnrich() - }, - ) + override fun assertEnrichedEvent(event: SentryEvent) { + val mainThread = event.threads!!.first() + assertEquals("main", mainThread.name) + assertEquals(1, mainThread.id) + assertEquals("Blocked", mainThread.state) + assertEquals(true, mainThread.isCrashed) + assertEquals(true, mainThread.isMain) + assertEquals("0x0d3a2f0a", mainThread.heldLocks!!.values.first().address) + assertEquals(5, mainThread.heldLocks!!.values.first().threadId) + + val lastFrame = mainThread.stacktrace!!.frames!!.last() + assertEquals("io.sentry.samples.android.MainActivity$2", lastFrame.module) + assertEquals("MainActivity.java", lastFrame.filename) + assertEquals("run", lastFrame.function) + assertEquals(177, lastFrame.lineno) + assertEquals(true, lastFrame.isInApp) + + val otherThread = event.threads!![1] + assertEquals("perfetto_hprof_listener", otherThread.name) + assertEquals(7, otherThread.id) + assertEquals("Native", otherThread.state) + assertEquals(false, otherThread.isCrashed) + assertEquals(false, otherThread.isMain) + + val firstFrame = otherThread.stacktrace!!.frames!!.first() + assertEquals("/apex/com.android.runtime/lib64/bionic/libc.so", firstFrame.`package`) + assertEquals("__start_thread", firstFrame.function) + assertEquals(64, firstFrame.lineno) + assertEquals("0x00000000000530b8", firstFrame.instructionAddr) + assertEquals("native", firstFrame.platform) + assertEquals("rel:741f3301-bbb0-b92c-58bd-c15282b8ec7b", firstFrame.addrMode) + + val image = + event.debugMeta?.images?.find { it.debugId == "741f3301-bbb0-b92c-58bd-c15282b8ec7b" } + assertNotNull(image) + assertEquals("/apex/com.android.runtime/lib64/bionic/libc.so", image.codeFile) } @Test fun `when latest ANR has foreground importance, sets abnormal mechanism to anr_foreground`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp) + val integration = + fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp, sessionTrackingEnabled = true) fixture.addAppExitInfo( timestamp = newTimestamp, importance = ActivityManager.RunningAppProcessInfo.IMPORTANCE_FOREGROUND, @@ -344,145 +180,9 @@ class AnrV2IntegrationTest { ) } - @Test - fun `waits for ANR events to be flushed on disk`() { - val integration = - fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp, flushTimeoutMillis = 500L) - fixture.addAppExitInfo(timestamp = newTimestamp) - - whenever(fixture.scopes.captureEvent(any(), any())).thenAnswer { invocation -> - val hint = HintUtils.getSentrySdkHint(invocation.getArgument(1)) as DiskFlushNotification - thread { - Thread.sleep(200L) - hint.markFlushed() - } - SentryId() - } - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes).captureEvent(any(), anyOrNull()) - // shouldn't fall into timed out state, because we marked event as flushed on another thread - verify(fixture.logger, never()) - .log( - any(), - argThat { startsWith("Timed out waiting to flush ANR event to disk.") }, - any(), - ) - } - - @Test - fun `when latest ANR event was dropped, does not block flushing`() { - val integration = - fixture.getSut( - tmpDir, - lastReportedAnrTimestamp = oldTimestamp, - lastEventId = SentryId.EMPTY_ID, - ) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes).captureEvent(any(), anyOrNull()) - // we do not call markFlushed, hence it should time out waiting for flush, but because - // we drop the event, it should not even come to this if-check - verify(fixture.logger, never()) - .log( - any(), - argThat { startsWith("Timed out waiting to flush ANR event to disk.") }, - any(), - ) - } - - @Test - fun `historical ANRs are reported non-enriched`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp - 2 * 60 * 1000) - fixture.addAppExitInfo(timestamp = newTimestamp - 1 * 60 * 1000) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, times(2)) - .captureEvent( - any(), - argThat { - val hint = HintUtils.getSentrySdkHint(this) - !(hint as AnrV2Hint).shouldEnrich() - }, - ) - } - - @Test - fun `when historical ANRs flag is disabled, does not report`() { - val integration = - fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp, reportHistoricalAnrs = false) - fixture.addAppExitInfo(timestamp = newTimestamp - 2 * 60 * 1000) - fixture.addAppExitInfo(timestamp = newTimestamp - 1 * 60 * 1000) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - // only the latest anr is reported which should be enrichable - verify(fixture.scopes, atMost(1)) - .captureEvent( - any(), - argThat { - val hint = HintUtils.getSentrySdkHint(this) - (hint as AnrV2Hint).shouldEnrich() - }, - ) - } - - @Test - fun `historical ANRs are reported in reverse order to keep track of last reported ANR in a marker file`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp) - // robolectric uses addFirst when adding exit infos, so the last one here will be the first on - // the list - fixture.addAppExitInfo(timestamp = newTimestamp - TimeUnit.DAYS.toMillis(2)) - fixture.addAppExitInfo(timestamp = newTimestamp - TimeUnit.DAYS.toMillis(1)) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - // the order is reverse here, so the oldest ANR will be reported first to keep track of - // last reported ANR in a marker file - inOrder(fixture.scopes) { - verify(fixture.scopes) - .captureEvent( - argThat { timestamp.time == newTimestamp - TimeUnit.DAYS.toMillis(2) }, - anyOrNull(), - ) - verify(fixture.scopes) - .captureEvent( - argThat { timestamp.time == newTimestamp - TimeUnit.DAYS.toMillis(1) }, - anyOrNull(), - ) - verify(fixture.scopes) - .captureEvent(argThat { timestamp.time == newTimestamp }, anyOrNull()) - } - } - - @Test - fun `ANR timestamp is passed with the hint`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes) - .captureEvent( - any(), - argThat { - val hint = HintUtils.getSentrySdkHint(this) - (hint as AnrV2Hint).timestamp() == newTimestamp - }, - ) - } - @Test fun `abnormal mechanism is passed with the hint`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp) + val integration = fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp) fixture.addAppExitInfo(timestamp = newTimestamp) integration.register(fixture.scopes, fixture.options) @@ -498,108 +198,17 @@ class AnrV2IntegrationTest { } @Test - fun `awaits for previous session flush if cache is EnvelopeCache`() { - val integration = - fixture.getSut( - tmpDir, - lastReportedAnrTimestamp = oldTimestamp, - sessionFlushTimeoutMillis = 500L, - ) - fixture.addAppExitInfo(timestamp = newTimestamp) - - thread { - Thread.sleep(200L) - val sessionHint = HintUtils.createWithTypeCheckHint(SessionStartHint()) - fixture.options.envelopeDiskCache.storeEnvelope( - SentryEnvelope(SentryId.EMPTY_ID, null, emptyList()), - sessionHint, - ) - } - - integration.register(fixture.scopes, fixture.options) - - // we store envelope with StartSessionHint on different thread after some delay, which - // triggers the previous session flush, so no timeout - verify(fixture.logger, never()) - .log( - any(), - argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, - any(), - ) - } - - @Test - fun `does not await for previous session flush, if session tracking is disabled`() { - val integration = - fixture.getSut( - tmpDir, - lastReportedAnrTimestamp = oldTimestamp, - sessionFlushTimeoutMillis = 500L, - sessionTrackingEnabled = false, - ) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.logger, never()) - .log( - any(), - argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, - any(), - ) - verify(fixture.scopes).captureEvent(any(), any()) - } - - @Test - fun `flushes previous session latch, if timed out waiting`() { + fun `attaches plain thread dump, if enabled`() { val integration = fixture.getSut( tmpDir, - lastReportedAnrTimestamp = oldTimestamp, - sessionFlushTimeoutMillis = 500L, + lastReportedTimestamp = oldTimestamp, + extraOptions = { opts -> opts.isAttachAnrThreadDump = true }, ) fixture.addAppExitInfo(timestamp = newTimestamp) integration.register(fixture.scopes, fixture.options) - verify(fixture.logger) - .log( - any(), - argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, - any(), - ) - // should return true, because latch is 0 now - assertTrue((fixture.options.envelopeDiskCache as EnvelopeCache).waitPreviousSessionFlush()) - } - - @Test - fun `attaches plain thread dump, if enabled`() { - val integration = - fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp, attachAnrThreadDump = true) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - verify(fixture.scopes).captureEvent(any(), check { assertNotNull(it.threadDump) }) } - - @Test - fun `when traceInputStream is null, does not report ANR`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp, addTrace = false) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when traceInputStream has bad data, does not report ANR`() { - val integration = fixture.getSut(tmpDir, lastReportedAnrTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp, addBadTrace = true) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } } diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/ApplicationExitIntegrationTestBase.kt b/sentry-android-core/src/test/java/io/sentry/android/core/ApplicationExitIntegrationTestBase.kt new file mode 100644 index 00000000000..4f2533d4268 --- /dev/null +++ b/sentry-android-core/src/test/java/io/sentry/android/core/ApplicationExitIntegrationTestBase.kt @@ -0,0 +1,435 @@ +package io.sentry.android.core + +import android.app.ActivityManager +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import io.sentry.Hint +import io.sentry.ILogger +import io.sentry.IScopes +import io.sentry.Integration +import io.sentry.SentryEnvelope +import io.sentry.SentryEvent +import io.sentry.SentryLevel +import io.sentry.cache.EnvelopeCache +import io.sentry.hints.DiskFlushNotification +import io.sentry.hints.SessionStartHint +import io.sentry.protocol.SentryId +import io.sentry.test.ImmediateExecutorService +import io.sentry.util.HintUtils +import java.io.File +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import org.mockito.kotlin.any +import org.mockito.kotlin.anyOrNull +import org.mockito.kotlin.argThat +import org.mockito.kotlin.atMost +import org.mockito.kotlin.check +import org.mockito.kotlin.inOrder +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.robolectric.shadow.api.Shadow +import org.robolectric.shadows.ShadowActivityManager + +abstract class ApplicationExitIntegrationTestBase { + + protected abstract val config: IntegrationTestConfig + + @get:Rule val tmpDir = TemporaryFolder() + + protected val fixture: ApplicationExitTestFixture by lazy { + ApplicationExitTestFixture(config) + } + protected val oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(10) + protected val newTimestamp = oldTimestamp + TimeUnit.DAYS.toMillis(5) + + @BeforeTest + fun `set up`() { + val context = ApplicationProvider.getApplicationContext() + fixture.init(context) + } + + @Test + fun `when cacheDir is not set, does not process historical exits`() { + val integration = fixture.getSut(null, useImmediateExecutorService = false) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.options.executorService, never()).submit(any()) + } + + @Test + fun `when integration is not enabled, does not process historical exits`() { + val integration = fixture.getSut(tmpDir, enabled = false, useImmediateExecutorService = false) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.options.executorService, never()).submit(any()) + } + + @Test + fun `when historical exit list is empty, does not process historical exits`() { + val integration = fixture.getSut(tmpDir) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when there are no matching exits, does not capture events`() { + val integration = fixture.getSut(tmpDir) + fixture.addAppExitInfo(reason = null) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when latest exit is older than 90 days, does not capture events`() { + val oldTimestamp = + System.currentTimeMillis() - + ApplicationExitInfoHistoryDispatcher.NINETY_DAYS_THRESHOLD - + TimeUnit.DAYS.toMillis(2) + val integration = fixture.getSut(tmpDir) + fixture.addAppExitInfo(timestamp = oldTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when latest exit has already been reported, does not capture events`() { + val integration = fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = oldTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when no exits have ever been reported, captures events`() { + val integration = fixture.getSut(tmpDir, lastReportedTimestamp = null) + fixture.addAppExitInfo(timestamp = oldTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when latest exit has not been reported, captures event with enriching`() { + val integration = fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes) + .captureEvent( + check { event -> + assertEquals(event.timestamp!!.time, newTimestamp) + assertEquals(event.level, SentryLevel.FATAL) + assertEnrichedEvent(event) + }, + argThat { + val hint = config.hintAccessors.cast(HintUtils.getSentrySdkHint(this)) + config.hintAccessors.shouldEnrich(hint) + }, + ) + } + + @Test + fun `waits for events to be flushed on disk`() { + val integration = + fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp, flushTimeoutMillis = 500L) + fixture.addAppExitInfo(timestamp = newTimestamp) + + whenever(fixture.scopes.captureEvent(any(), any())).thenAnswer { invocation -> + val hint = HintUtils.getSentrySdkHint(invocation.getArgument(1)) as DiskFlushNotification + thread { + Thread.sleep(200L) + hint.markFlushed() + } + SentryId() + } + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes).captureEvent(any(), anyOrNull()) + verify(fixture.logger, never()) + .log(any(), argThat { startsWith(config.flushLogPrefix) }, any()) + } + + @Test + fun `when latest event was dropped, does not block flushing`() { + val integration = + fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp, lastEventId = SentryId.EMPTY_ID) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes).captureEvent(any(), anyOrNull()) + verify(fixture.logger, never()) + .log(any(), argThat { startsWith(config.flushLogPrefix) }, any()) + } + + @Test + fun `historical exits are reported non-enriched`() { + val integration = fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp - 2 * 60 * 1000) + fixture.addAppExitInfo(timestamp = newTimestamp - 1 * 60 * 1000) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, times(2)) + .captureEvent( + any(), + argThat { + val hint = config.hintAccessors.cast(HintUtils.getSentrySdkHint(this)) + !config.hintAccessors.shouldEnrich(hint) + }, + ) + } + + @Test + fun `when historical flag is disabled, does not report`() { + val integration = + fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp, reportHistorical = false) + fixture.addAppExitInfo(timestamp = newTimestamp - 2 * 60 * 1000) + fixture.addAppExitInfo(timestamp = newTimestamp - 1 * 60 * 1000) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, atMost(1)) + .captureEvent( + any(), + argThat { + val hint = config.hintAccessors.cast(HintUtils.getSentrySdkHint(this)) + config.hintAccessors.shouldEnrich(hint) + }, + ) + } + + @Test + fun `historical exits are reported in reverse order to keep track of last reported exit in a marker file`() { + val integration = fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp - TimeUnit.DAYS.toMillis(2)) + fixture.addAppExitInfo(timestamp = newTimestamp - TimeUnit.DAYS.toMillis(1)) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + inOrder(fixture.scopes) { + verify(fixture.scopes) + .captureEvent( + argThat { timestamp.time == newTimestamp - TimeUnit.DAYS.toMillis(2) }, + anyOrNull(), + ) + verify(fixture.scopes) + .captureEvent( + argThat { timestamp.time == newTimestamp - TimeUnit.DAYS.toMillis(1) }, + anyOrNull(), + ) + verify(fixture.scopes) + .captureEvent(argThat { timestamp.time == newTimestamp }, anyOrNull()) + } + } + + @Test + fun `timestamp is passed with the hint`() { + val integration = fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes) + .captureEvent( + any(), + argThat { + val hint = config.hintAccessors.cast(HintUtils.getSentrySdkHint(this)) + config.hintAccessors.timestamp(hint) == newTimestamp + }, + ) + } + + @Test + fun `awaits for previous session flush if cache is EnvelopeCache`() { + val integration = + fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp, sessionFlushTimeoutMillis = 500L) + fixture.addAppExitInfo(timestamp = newTimestamp) + + thread { + Thread.sleep(200L) + val sessionHint = HintUtils.createWithTypeCheckHint(SessionStartHint()) + fixture.options.envelopeDiskCache.storeEnvelope( + SentryEnvelope(SentryId.EMPTY_ID, null, emptyList()), + sessionHint, + ) + } + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.logger, never()) + .log( + any(), + argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, + any(), + ) + } + + @Test + fun `does not await for previous session flush, if session tracking is disabled`() { + val integration = + fixture.getSut( + tmpDir, + lastReportedTimestamp = oldTimestamp, + sessionFlushTimeoutMillis = 500L, + sessionTrackingEnabled = false, + ) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.logger, never()) + .log( + any(), + argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, + any(), + ) + verify(fixture.scopes).captureEvent(any(), any()) + } + + @Test + fun `flushes previous session latch, if timed out waiting`() { + val integration = + fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp, sessionFlushTimeoutMillis = 500L) + fixture.addAppExitInfo(timestamp = newTimestamp) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.logger) + .log( + any(), + argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, + any(), + ) + assertTrue((fixture.options.envelopeDiskCache as EnvelopeCache).waitPreviousSessionFlush()) + } + + @Test + fun `when traceInputStream is null, does not report`() { + val integration = fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp, addTrace = false) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + @Test + fun `when traceInputStream has bad data, does not report`() { + val integration = fixture.getSut(tmpDir, lastReportedTimestamp = oldTimestamp) + fixture.addAppExitInfo(timestamp = newTimestamp, addBadTrace = true) + + integration.register(fixture.scopes, fixture.options) + + verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + } + + protected open fun assertEnrichedEvent(event: SentryEvent) {} + + protected data class HintAccessors( + val cast: (Any?) -> THint, + val shouldEnrich: (THint) -> Boolean, + val timestamp: (THint) -> Long, + ) + + protected data class IntegrationTestConfig( + val setEnabledFlag: SentryAndroidOptions.(Boolean) -> Unit, + val setReportHistoricalFlag: SentryAndroidOptions.(Boolean) -> Unit, + val createIntegration: (Context) -> Integration, + val lastReportedFileName: String, + val defaultExitReason: Int, + val hintAccessors: HintAccessors, + val addExitInfo: + ApplicationExitTestFixture.( + reason: Int?, timestamp: Long?, importance: Int?, addTrace: Boolean, addBadTrace: Boolean, + ) -> Unit, + val flushLogPrefix: String, + ) + + protected class ApplicationExitTestFixture( + private val config: IntegrationTestConfig + ) { + lateinit var context: Context + lateinit var shadowActivityManager: ShadowActivityManager + lateinit var lastReportedFile: File + + val options = SentryAndroidOptions() + val scopes = mock() + val logger = mock() + + fun init(appContext: Context) { + context = appContext + val activityManager = context.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager? + shadowActivityManager = Shadow.extract(activityManager) + } + + fun getSut( + dir: TemporaryFolder?, + useImmediateExecutorService: Boolean = true, + enabled: Boolean = true, + flushTimeoutMillis: Long = 0L, + sessionFlushTimeoutMillis: Long = 0L, + lastReportedTimestamp: Long? = null, + lastEventId: SentryId = SentryId(), + sessionTrackingEnabled: Boolean = true, + reportHistorical: Boolean = true, + extraOptions: (SentryAndroidOptions) -> Unit = {}, + ): Integration { + options.run { + setLogger(this@ApplicationExitTestFixture.logger) + isDebug = true + cacheDirPath = dir?.newFolder()?.absolutePath + executorService = if (useImmediateExecutorService) ImmediateExecutorService() else mock() + config.setEnabledFlag(this, enabled) + this.flushTimeoutMillis = flushTimeoutMillis + this.sessionFlushTimeoutMillis = sessionFlushTimeoutMillis + this.isEnableAutoSessionTracking = sessionTrackingEnabled + config.setReportHistoricalFlag(this, reportHistorical) + addInAppInclude("io.sentry.samples") + setEnvelopeDiskCache(EnvelopeCache.create(this)) + extraOptions(this) + } + options.cacheDirPath?.let { cacheDir -> + lastReportedFile = File(cacheDir, config.lastReportedFileName) + lastReportedFile.writeText(lastReportedTimestamp.toString()) + } + whenever(scopes.captureEvent(any(), anyOrNull())).thenReturn(lastEventId) + return config.createIntegration(context) + } + + fun addAppExitInfo( + reason: Int? = config.defaultExitReason, + timestamp: Long? = null, + importance: Int? = null, + addTrace: Boolean = true, + addBadTrace: Boolean = false, + ) { + config.addExitInfo(this, reason, timestamp, importance, addTrace, addBadTrace) + } + } +} diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/TombstoneIntegrationTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/TombstoneIntegrationTest.kt index 8c2f47192ac..cb3988a6297 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/TombstoneIntegrationTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/TombstoneIntegrationTest.kt @@ -1,486 +1,76 @@ package io.sentry.android.core -import android.app.ActivityManager import android.app.ApplicationExitInfo -import android.content.Context -import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 -import io.sentry.Hint -import io.sentry.ILogger -import io.sentry.IScopes -import io.sentry.SentryEnvelope +import io.sentry.SentryEvent import io.sentry.SentryLevel import io.sentry.android.core.TombstoneIntegration.TombstoneHint import io.sentry.android.core.cache.AndroidEnvelopeCache -import io.sentry.cache.EnvelopeCache -import io.sentry.hints.DiskFlushNotification -import io.sentry.hints.SessionStartHint -import io.sentry.protocol.SentryId -import io.sentry.test.ImmediateExecutorService -import io.sentry.util.HintUtils import java.io.File -import java.util.concurrent.TimeUnit -import kotlin.concurrent.thread -import kotlin.test.BeforeTest -import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull import kotlin.test.assertTrue -import org.junit.Rule -import org.junit.rules.TemporaryFolder import org.junit.runner.RunWith -import org.mockito.kotlin.any -import org.mockito.kotlin.anyOrNull -import org.mockito.kotlin.argThat -import org.mockito.kotlin.atMost -import org.mockito.kotlin.check -import org.mockito.kotlin.inOrder -import org.mockito.kotlin.mock -import org.mockito.kotlin.never import org.mockito.kotlin.spy -import org.mockito.kotlin.times -import org.mockito.kotlin.verify import org.mockito.kotlin.whenever import org.robolectric.annotation.Config -import org.robolectric.shadow.api.Shadow -import org.robolectric.shadows.ShadowActivityManager import org.robolectric.shadows.ShadowActivityManager.ApplicationExitInfoBuilder @RunWith(AndroidJUnit4::class) @Config(sdk = [30]) -class TombstoneIntegrationTest { - @get:Rule val tmpDir = TemporaryFolder() - - class Fixture { - lateinit var context: Context - lateinit var shadowActivityManager: ShadowActivityManager - lateinit var lastReportedTombstoneFile: File - - val options = SentryAndroidOptions() - val scopes = mock() - val logger = mock() - - fun getSut( - dir: TemporaryFolder?, - useImmediateExecutorService: Boolean = true, - isTombstoneEnabled: Boolean = true, - flushTimeoutMillis: Long = 0L, - sessionFlushTimeoutMillis: Long = 0L, - lastReportedTombstoneTimestamp: Long? = null, - lastEventId: SentryId = SentryId(), - sessionTrackingEnabled: Boolean = true, - reportHistoricalTombstones: Boolean = true, - ): TombstoneIntegration { - options.run { - setLogger(this@Fixture.logger) - isDebug = true - cacheDirPath = dir?.newFolder()?.absolutePath - executorService = if (useImmediateExecutorService) ImmediateExecutorService() else mock() - this.isTombstoneEnabled = isTombstoneEnabled - this.flushTimeoutMillis = flushTimeoutMillis - this.sessionFlushTimeoutMillis = sessionFlushTimeoutMillis - this.isEnableAutoSessionTracking = sessionTrackingEnabled - this.isReportHistoricalTombstones = reportHistoricalTombstones - addInAppInclude("io.sentry.samples") - setEnvelopeDiskCache(EnvelopeCache.create(this)) - } - options.cacheDirPath?.let { cacheDir -> - lastReportedTombstoneFile = File(cacheDir, AndroidEnvelopeCache.LAST_TOMBSTONE_REPORT) - lastReportedTombstoneFile.writeText(lastReportedTombstoneTimestamp.toString()) - } - whenever(scopes.captureEvent(any(), anyOrNull())).thenReturn(lastEventId) - return TombstoneIntegration(context) - } - - fun addAppExitInfo( - reason: Int? = ApplicationExitInfo.REASON_CRASH_NATIVE, - timestamp: Long? = null, - importance: Int? = null, - addTrace: Boolean = true, - addBadTrace: Boolean = false, - ) { - val builder = ApplicationExitInfoBuilder.newBuilder() - if (reason != null) { - builder.setReason(reason) - } - if (timestamp != null) { - builder.setTimestamp(timestamp) - } - if (importance != null) { - builder.setImportance(importance) - } - val exitInfo = - spy(builder.build()) { - if (!addTrace) { - return - } - if (addBadTrace) { - whenever(mock.traceInputStream).thenReturn("XXXXX".byteInputStream()) - } else { - whenever(mock.traceInputStream) - .thenReturn(File("src/test/resources/tombstone.pb").inputStream()) - } - } - shadowActivityManager.addApplicationExitInfo(exitInfo) - } - } - - private val fixture = Fixture() - private val oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(10) - private val newTimestamp = oldTimestamp + TimeUnit.DAYS.toMillis(5) - - @BeforeTest - fun `set up`() { - fixture.context = ApplicationProvider.getApplicationContext() - val activityManager = - fixture.context.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager? - fixture.shadowActivityManager = Shadow.extract(activityManager) - } - - @Test - fun `when cacheDir is not set, does not process historical exits`() { - val integration = fixture.getSut(null, useImmediateExecutorService = false) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.options.executorService, never()).submit(any()) - } - - @Test - fun `when tombstone tracking is not enabled, does not process historical exits`() { - val integration = - fixture.getSut(tmpDir, isTombstoneEnabled = false, useImmediateExecutorService = false) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.options.executorService, never()).submit(any()) - } - - @Test - fun `when historical exit list is empty, does not process historical exits`() { - val integration = fixture.getSut(tmpDir) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when there are no tombstones in historical exits, does not capture events`() { - val integration = fixture.getSut(tmpDir) - fixture.addAppExitInfo(reason = null) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when latest Tombstone is older than 90 days, does not capture events`() { - val oldTimestamp = - System.currentTimeMillis() - - ApplicationExitInfoHistoryDispatcher.NINETY_DAYS_THRESHOLD - - TimeUnit.DAYS.toMillis(2) - val integration = fixture.getSut(tmpDir) - fixture.addAppExitInfo(timestamp = oldTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when latest Tombstone has already been reported, does not capture events`() { - val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = oldTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when no Tombstones have ever been reported, captures events`() { - val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = null) - fixture.addAppExitInfo(timestamp = oldTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when latest Tombstone has not been reported, captures event with enriching`() { - val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes) - .captureEvent( - check { - assertEquals(newTimestamp, it.timestamp.time) - assertEquals(SentryLevel.FATAL, it.level) - assertEquals("native", it.platform) - - val crashedThreadId = 21891.toLong() - assertEquals(crashedThreadId, it.exceptions!![0].threadId) - val crashedThread = it.threads!!.find { thread -> thread.id == crashedThreadId } - assertEquals("samples.android", crashedThread!!.name) - assertTrue(crashedThread.isCrashed!!) - - val image = - it.debugMeta?.images?.find { image -> - image.debugId == "f60b4b74005f33fb3ef3b98aa4546008" +class TombstoneIntegrationTest : ApplicationExitIntegrationTestBase() { + + override val config = + IntegrationTestConfig( + setEnabledFlag = { isTombstoneEnabled = it }, + setReportHistoricalFlag = { isReportHistoricalTombstones = it }, + createIntegration = { context -> TombstoneIntegration(context) }, + lastReportedFileName = AndroidEnvelopeCache.LAST_TOMBSTONE_REPORT, + defaultExitReason = ApplicationExitInfo.REASON_CRASH_NATIVE, + hintAccessors = + HintAccessors( + cast = { it as TombstoneHint }, + shouldEnrich = { it.shouldEnrich() }, + timestamp = { it.timestamp() }, + ), + addExitInfo = { reason, timestamp, importance, addTrace, addBadTrace -> + val builder = ApplicationExitInfoBuilder.newBuilder() + reason?.let { builder.setReason(it) } + timestamp?.let { builder.setTimestamp(it) } + importance?.let { builder.setImportance(it) } + val exitInfo = + spy(builder.build()) { + if (!addTrace) { + return@spy } - assertNotNull(image) - assertEquals("/system/lib64/libcompiler_rt.so", image.codeFile) - assertEquals("0x764c32a000", image.imageAddr) - assertEquals(32768, image.imageSize) - }, - argThat { - val hint = HintUtils.getSentrySdkHint(this) - (hint as TombstoneHint).shouldEnrich() - }, - ) - } - - @Test - fun `waits for Tombstone events to be flushed on disk`() { - val integration = - fixture.getSut( - tmpDir, - lastReportedTombstoneTimestamp = oldTimestamp, - flushTimeoutMillis = 500L, - ) - fixture.addAppExitInfo(timestamp = newTimestamp) - - whenever(fixture.scopes.captureEvent(any(), any())).thenAnswer { invocation -> - val hint = HintUtils.getSentrySdkHint(invocation.getArgument(1)) as DiskFlushNotification - thread { - Thread.sleep(200L) - hint.markFlushed() - } - SentryId() - } - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes).captureEvent(any(), anyOrNull()) - // shouldn't fall into timed out state, because we marked event as flushed on another thread - verify(fixture.logger, never()) - .log( - any(), - argThat { startsWith("Timed out waiting to flush Tombstone event to disk.") }, - any(), - ) - } - - @Test - fun `when latest Tombstone event was dropped, does not block flushing`() { - val integration = - fixture.getSut( - tmpDir, - lastReportedTombstoneTimestamp = oldTimestamp, - lastEventId = SentryId.EMPTY_ID, - ) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes).captureEvent(any(), anyOrNull()) - // we do not call markFlushed, hence it should time out waiting for flush, but because - // we drop the event, it should not even come to this if-check - verify(fixture.logger, never()) - .log( - any(), - argThat { startsWith("Timed out waiting to flush Tombstone event to disk.") }, - any(), - ) - } - - @Test - fun `historical Tombstones are reported non-enriched`() { - val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp - 2 * 60 * 1000) - fixture.addAppExitInfo(timestamp = newTimestamp - 1 * 60 * 1000) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, times(2)) - .captureEvent( - any(), - argThat { - val hint = HintUtils.getSentrySdkHint(this) - !(hint as TombstoneHint).shouldEnrich() - }, - ) - } - - @Test - fun `when historical Tombstones flag is disabled, does not report`() { - val integration = - fixture.getSut( - tmpDir, - lastReportedTombstoneTimestamp = oldTimestamp, - reportHistoricalTombstones = false, - ) - fixture.addAppExitInfo(timestamp = newTimestamp - 2 * 60 * 1000) - fixture.addAppExitInfo(timestamp = newTimestamp - 1 * 60 * 1000) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - // only the latest tombstone is reported which should be enrichable - verify(fixture.scopes, atMost(1)) - .captureEvent( - any(), - argThat { - val hint = HintUtils.getSentrySdkHint(this) - (hint as TombstoneHint).shouldEnrich() - }, - ) - } - - @Test - fun `historical Tombstones are reported in reverse order to keep track of last reported Tombstone in a marker file`() { - val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) - // robolectric uses addFirst when adding exit infos, so the last one here will be the first on - // the list - fixture.addAppExitInfo(timestamp = newTimestamp - TimeUnit.DAYS.toMillis(2)) - fixture.addAppExitInfo(timestamp = newTimestamp - TimeUnit.DAYS.toMillis(1)) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - // the order is reverse here, so the oldest Tombstone will be reported first to keep track of - // last reported Tombstone in a marker file - inOrder(fixture.scopes) { - verify(fixture.scopes) - .captureEvent( - argThat { timestamp.time == newTimestamp - TimeUnit.DAYS.toMillis(2) }, - anyOrNull(), - ) - verify(fixture.scopes) - .captureEvent( - argThat { timestamp.time == newTimestamp - TimeUnit.DAYS.toMillis(1) }, - anyOrNull(), - ) - verify(fixture.scopes) - .captureEvent(argThat { timestamp.time == newTimestamp }, anyOrNull()) - } - } - - @Test - fun `Tombstone timestamp is passed with the hint`() { - val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes) - .captureEvent( - any(), - argThat { - val hint = HintUtils.getSentrySdkHint(this) - (hint as TombstoneHint).timestamp() == newTimestamp - }, - ) - } - - @Test - fun `awaits for previous session flush if cache is EnvelopeCache`() { - val integration = - fixture.getSut( - tmpDir, - lastReportedTombstoneTimestamp = oldTimestamp, - sessionFlushTimeoutMillis = 500L, - ) - fixture.addAppExitInfo(timestamp = newTimestamp) - - thread { - Thread.sleep(200L) - val sessionHint = HintUtils.createWithTypeCheckHint(SessionStartHint()) - fixture.options.envelopeDiskCache.storeEnvelope( - SentryEnvelope(SentryId.EMPTY_ID, null, emptyList()), - sessionHint, - ) - } - - integration.register(fixture.scopes, fixture.options) - - // we store envelope with StartSessionHint on different thread after some delay, which - // triggers the previous session flush, so no timeout - verify(fixture.logger, never()) - .log( - any(), - argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, - any(), - ) - } - - @Test - fun `does not await for previous session flush, if session tracking is disabled`() { - val integration = - fixture.getSut( - tmpDir, - lastReportedTombstoneTimestamp = oldTimestamp, - sessionFlushTimeoutMillis = 500L, - sessionTrackingEnabled = false, - ) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.logger, never()) - .log( - any(), - argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, - any(), - ) - verify(fixture.scopes).captureEvent(any(), any()) - } - - @Test - fun `flushes previous session latch, if timed out waiting`() { - val integration = - fixture.getSut( - tmpDir, - lastReportedTombstoneTimestamp = oldTimestamp, - sessionFlushTimeoutMillis = 500L, - ) - fixture.addAppExitInfo(timestamp = newTimestamp) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.logger) - .log( - any(), - argThat { startsWith("Timed out waiting to flush previous session to its own file.") }, - any(), - ) - // should return true, because latch is 0 now - assertTrue((fixture.options.envelopeDiskCache as EnvelopeCache).waitPreviousSessionFlush()) - } - - @Test - fun `when traceInputStream is null, does not report Tombstone`() { - val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp, addTrace = false) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) - } - - @Test - fun `when traceInputStream has bad data, does not report Tombstone`() { - val integration = fixture.getSut(tmpDir, lastReportedTombstoneTimestamp = oldTimestamp) - fixture.addAppExitInfo(timestamp = newTimestamp, addBadTrace = true) - - integration.register(fixture.scopes, fixture.options) - - verify(fixture.scopes, never()).captureEvent(any(), anyOrNull()) + if (addBadTrace) { + whenever(mock.traceInputStream).thenReturn("XXXXX".byteInputStream()) + } else { + whenever(mock.traceInputStream) + .thenReturn(File("src/test/resources/tombstone.pb").inputStream()) + } + } + shadowActivityManager.addApplicationExitInfo(exitInfo) + }, + flushLogPrefix = "Timed out waiting to flush Tombstone event to disk.", + ) + + override fun assertEnrichedEvent(event: SentryEvent) { + assertEquals(SentryLevel.FATAL, event.level) + assertEquals(newTimestamp, event.timestamp!!.time) + assertEquals("native", event.platform) + + val crashedThreadId = 21891L + assertEquals(crashedThreadId, event.exceptions!![0].threadId) + val crashedThread = event.threads!!.find { thread -> thread.id == crashedThreadId } + assertEquals("samples.android", crashedThread!!.name) + assertTrue(crashedThread.isCrashed!!) + + val image = + event.debugMeta?.images?.find { image -> image.debugId == "f60b4b74005f33fb3ef3b98aa4546008" } + assertNotNull(image) + assertEquals("/system/lib64/libcompiler_rt.so", image.codeFile) + assertEquals("0x764c32a000", image.imageAddr) + assertEquals(32768, image.imageSize) } } From 8499a1384cd255c44088ee84129edf9f6dbd362f Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Mon, 15 Dec 2025 12:35:32 +0100 Subject: [PATCH 28/35] remove obsolete TODO RE: mechanism change --- .../android/core/internal/tombstone/TombstoneParser.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index 60f9376dfa6..e884a5d52b4 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -140,12 +140,6 @@ private static Mechanism createMechanismFromSignalInfo( final Mechanism mechanism = new Mechanism(); // this follows the current processing triggers strictly, changing any of these // alters grouping and name (long-term we might want to have a tombstone mechanism) - // TODO: if we align this with ANRv2 this would be overwritten in a BackfillingEventProcessor as - // `ApplicationExitInfo` not sure what the right call is. `ApplicationExitInfo` is - // certainly correct. But `signalhandler` isn't wrong either, since all native crashes - // retrieved via `REASON_CRASH_NATIVE` will be signals. I am not sure what the side-effect - // in ingestion/processing will be if we change the mechanism, but initially i wanted to - // stay close to the Native SDK. mechanism.setType("signalhandler"); mechanism.setHandled(false); mechanism.setSynthetic(true); From 5bd19b066a0e221517ad49c74e624e52b14059d2 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Mon, 15 Dec 2025 13:01:54 +0100 Subject: [PATCH 29/35] clarify open TODO RE: AbnormalExit discriminator for ANR hints --- .../core/ApplicationExitInfoEventProcessor.java | 5 ++--- sentry/src/main/java/io/sentry/hints/AbnormalExit.java | 10 +++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java index 0f3b8c3a113..ba32c4976c2 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java @@ -682,9 +682,8 @@ private final class AnrHintEnricher implements HintEnricher { @Override public boolean supports(@NotNull Object hint) { - // TODO: not sure about this. all tests are written with AbnormalExit, but enrichment changes - // are ANR-specific. I called it AnrHintEnricher because that is what it does, but it - // actually triggers on AbnormalExit. Let me know what makes most sense. + // While this is specifically an ANR enricher we discriminate enrichment application + // on the broader AbnormalExit hints for now. return hint instanceof AbnormalExit; } diff --git a/sentry/src/main/java/io/sentry/hints/AbnormalExit.java b/sentry/src/main/java/io/sentry/hints/AbnormalExit.java index f9e67bc7607..bf63f98ea6f 100644 --- a/sentry/src/main/java/io/sentry/hints/AbnormalExit.java +++ b/sentry/src/main/java/io/sentry/hints/AbnormalExit.java @@ -2,7 +2,15 @@ import org.jetbrains.annotations.Nullable; -/** Marker interface for Sessions experiencing abnormal status */ +/** + * Marker interface for Sessions experiencing abnormal status. + * + *

    Note: While this interface applies to the broad category of abnormal exits (meaning any + * exits that weren't classified as normal terminations or crashes) it currently is exclusively used + * as a hint marker for Android ANRs (both watchdog and ApplicationExitInfo based). If additional + * categories of abnormal exits were introduced, all instances of discriminator code (`instanceof + * AbnormalExit`) should be carefully reviewed for ANR specifics accidentally being applied. + */ public interface AbnormalExit { /** What was the mechanism this Session has abnormal'ed with */ From 36fa71ecf319dd5b1b539f7de91a4a674e044bf9 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Mon, 15 Dec 2025 13:40:18 +0100 Subject: [PATCH 30/35] refactor instructionAddressAdjustment from String to enum --- .../internal/tombstone/TombstoneParser.java | 2 +- sentry/api/sentry.api | 20 +++++++++- .../main/java/io/sentry/JsonSerializer.java | 3 ++ .../io/sentry/protocol/SentryStackTrace.java | 40 +++++++++++++++---- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index e884a5d52b4..04fc3541fe2 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -104,7 +104,7 @@ private static SentryStackTrace createStackTrace(@NonNull final TombstoneProtos. // `libunwindstack` used for tombstones already applies instruction address adjustment: // https://android.googlesource.com/platform/system/unwinding/+/refs/heads/main/libunwindstack/Regs.cpp#175 // prevent "processing" from doing it again. - stacktrace.setInstructionAddressAdjustment("none"); + stacktrace.setInstructionAddressAdjustment(SentryStackTrace.InstructionAddressAdjustment.NONE); final Map registers = new HashMap<>(); for (TombstoneProtos.Register register : thread.getRegistersList()) { diff --git a/sentry/api/sentry.api b/sentry/api/sentry.api index 4f35abde3e7..f2e9c2fa4f7 100644 --- a/sentry/api/sentry.api +++ b/sentry/api/sentry.api @@ -6233,13 +6233,13 @@ public final class io/sentry/protocol/SentryStackTrace : io/sentry/JsonSerializa public fun ()V public fun (Ljava/util/List;)V public fun getFrames ()Ljava/util/List; - public fun getInstructionAddressAdjustment ()Ljava/lang/String; + public fun getInstructionAddressAdjustment ()Lio/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment; public fun getRegisters ()Ljava/util/Map; public fun getSnapshot ()Ljava/lang/Boolean; public fun getUnknown ()Ljava/util/Map; public fun serialize (Lio/sentry/ObjectWriter;Lio/sentry/ILogger;)V public fun setFrames (Ljava/util/List;)V - public fun setInstructionAddressAdjustment (Ljava/lang/String;)V + public fun setInstructionAddressAdjustment (Lio/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment;)V public fun setRegisters (Ljava/util/Map;)V public fun setSnapshot (Ljava/lang/Boolean;)V public fun setUnknown (Ljava/util/Map;)V @@ -6251,6 +6251,22 @@ public final class io/sentry/protocol/SentryStackTrace$Deserializer : io/sentry/ public synthetic fun deserialize (Lio/sentry/ObjectReader;Lio/sentry/ILogger;)Ljava/lang/Object; } +public final class io/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment : java/lang/Enum, io/sentry/JsonSerializable { + public static final field ALL Lio/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment; + public static final field ALL_BUT_FIRST Lio/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment; + public static final field AUTO Lio/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment; + public static final field NONE Lio/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment; + public fun serialize (Lio/sentry/ObjectWriter;Lio/sentry/ILogger;)V + public static fun valueOf (Ljava/lang/String;)Lio/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment; + public static fun values ()[Lio/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment; +} + +public final class io/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment$Deserializer : io/sentry/JsonDeserializer { + public fun ()V + public fun deserialize (Lio/sentry/ObjectReader;Lio/sentry/ILogger;)Lio/sentry/protocol/SentryStackTrace$InstructionAddressAdjustment; + public synthetic fun deserialize (Lio/sentry/ObjectReader;Lio/sentry/ILogger;)Ljava/lang/Object; +} + public final class io/sentry/protocol/SentryStackTrace$JsonKeys { public static final field FRAMES Ljava/lang/String; public static final field INSTRUCTION_ADDRESS_ADJUSTMENT Ljava/lang/String; diff --git a/sentry/src/main/java/io/sentry/JsonSerializer.java b/sentry/src/main/java/io/sentry/JsonSerializer.java index 65e672bea13..c643fd2a696 100644 --- a/sentry/src/main/java/io/sentry/JsonSerializer.java +++ b/sentry/src/main/java/io/sentry/JsonSerializer.java @@ -87,6 +87,9 @@ public JsonSerializer(@NotNull SentryOptions options) { Device.DeviceOrientation.class, new Device.DeviceOrientation.Deserializer()); deserializersByClass.put(Feedback.class, new Feedback.Deserializer()); deserializersByClass.put(Gpu.class, new Gpu.Deserializer()); + deserializersByClass.put( + SentryStackTrace.InstructionAddressAdjustment.class, + new SentryStackTrace.InstructionAddressAdjustment.Deserializer()); deserializersByClass.put(MeasurementValue.class, new MeasurementValue.Deserializer()); deserializersByClass.put(Mechanism.class, new Mechanism.Deserializer()); deserializersByClass.put(Message.class, new Message.Deserializer()); diff --git a/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java b/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java index f04982a4f69..bc08f542b5f 100644 --- a/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java +++ b/sentry/src/main/java/io/sentry/protocol/SentryStackTrace.java @@ -10,6 +10,7 @@ import io.sentry.vendor.gson.stream.JsonToken; import java.io.IOException; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.jetbrains.annotations.NotNull; @@ -68,13 +69,12 @@ public final class SentryStackTrace implements JsonUnknown, JsonSerializable { /** * This value indicates if, and how, `instruction_addr` values in the stack frames need to be - * adjusted before they are symbolicated. TODO: should we make this an enum or is a string value - * fine? + * adjusted before they are symbolicated. * * @see SentryStackFrame#getInstructionAddr() * @see SentryStackFrame#setInstructionAddr(String) */ - private @Nullable String instructionAddressAdjustment; + private @Nullable InstructionAddressAdjustment instructionAddressAdjustment; @SuppressWarnings("unused") private @Nullable Map unknown; @@ -132,11 +132,12 @@ public void setUnknown(@Nullable Map unknown) { this.unknown = unknown; } - public @Nullable String getInstructionAddressAdjustment() { + public @Nullable InstructionAddressAdjustment getInstructionAddressAdjustment() { return instructionAddressAdjustment; } - public void setInstructionAddressAdjustment(@Nullable String instructionAddressAdjustment) { + public void setInstructionAddressAdjustment( + @Nullable InstructionAddressAdjustment instructionAddressAdjustment) { this.instructionAddressAdjustment = instructionAddressAdjustment; } @@ -161,7 +162,9 @@ public void serialize(final @NotNull ObjectWriter writer, final @NotNull ILogger writer.name(JsonKeys.SNAPSHOT).value(snapshot); } if (instructionAddressAdjustment != null) { - writer.name(JsonKeys.INSTRUCTION_ADDRESS_ADJUSTMENT).value(instructionAddressAdjustment); + writer + .name(JsonKeys.INSTRUCTION_ADDRESS_ADJUSTMENT) + .value(logger, instructionAddressAdjustment); } if (unknown != null) { for (String key : unknown.keySet()) { @@ -197,7 +200,8 @@ public static final class Deserializer implements JsonDeserializer { + @Override + public @NotNull SentryStackTrace.InstructionAddressAdjustment deserialize( + @NotNull ObjectReader reader, @NotNull ILogger logger) throws Exception { + return InstructionAddressAdjustment.valueOf(reader.nextString().toUpperCase(Locale.ROOT)); + } + } + } } From e13da214f2f1dcb339ae1397169c6be7ac565889 Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 16 Dec 2025 11:18:16 +0100 Subject: [PATCH 31/35] deduplicate hex formatting of addresses and TombstoneParser --- .../core/internal/tombstone/TombstoneParser.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index 04fc3541fe2..3d60f762b83 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -25,6 +25,10 @@ public class TombstoneParser implements Closeable { private final InputStream tombstoneStream; private final Map excTypeValueMap = new HashMap<>(); + private static String formatHex(long value) { + return String.format("0x%x", value); + } + public TombstoneParser(@NonNull final InputStream tombstoneStream) { this.tombstoneStream = tombstoneStream; @@ -94,7 +98,7 @@ private static SentryStackTrace createStackTrace(@NonNull final TombstoneProtos. final SentryStackFrame stackFrame = new SentryStackFrame(); stackFrame.setPackage(frame.getFileName()); stackFrame.setFunction(frame.getFunctionName()); - stackFrame.setInstructionAddr(String.format("0x%x", frame.getPc())); + stackFrame.setInstructionAddr(formatHex(frame.getPc())); frames.add(0, stackFrame); } @@ -108,7 +112,7 @@ private static SentryStackTrace createStackTrace(@NonNull final TombstoneProtos. final Map registers = new HashMap<>(); for (TombstoneProtos.Register register : thread.getRegistersList()) { - registers.put(register.getName(), String.format("0x%x", register.getU64())); + registers.put(register.getName(), formatHex(register.getU64())); } stacktrace.setRegisters(registers); @@ -196,7 +200,7 @@ private DebugMeta createDebugMeta(@NonNull final TombstoneProtos.Tombstone tombs image.setCodeId(module.getBuildId()); image.setCodeFile(module.getMappingName()); image.setDebugId(module.getBuildId()); - image.setImageAddr(String.format("0x%x", module.getBeginAddress())); + image.setImageAddr(formatHex(module.getBeginAddress())); image.setImageSize(module.getEndAddress() - module.getBeginAddress()); image.setType("elf"); From 37e43bb7bd2533ca79304d12726605b6bca36efe Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Tue, 16 Dec 2025 15:14:58 +0100 Subject: [PATCH 32/35] adapt tryEndPreviousSession() in EnvelopeCache to NativeCrashExit hint --- .../java/io/sentry/cache/EnvelopeCache.java | 57 ++++++++++++------- .../java/io/sentry/cache/EnvelopeCacheTest.kt | 48 ++++++++++++++++ 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java b/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java index 9cc1ca9cf63..8016118875a 100644 --- a/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java +++ b/sentry/src/main/java/io/sentry/cache/EnvelopeCache.java @@ -20,6 +20,7 @@ import io.sentry.Session; import io.sentry.UncaughtExceptionHandlerIntegration; import io.sentry.hints.AbnormalExit; +import io.sentry.hints.NativeCrashExit; import io.sentry.hints.SessionEnd; import io.sentry.hints.SessionStart; import io.sentry.transport.NoOpEnvelopeCache; @@ -119,10 +120,10 @@ private boolean storeInternal(final @NotNull SentryEnvelope envelope, final @Not } } - if (HintUtils.hasType(hint, AbnormalExit.class)) { + if (HintUtils.hasType(hint, AbnormalExit.class) + || HintUtils.hasType(hint, NativeCrashExit.class)) { tryEndPreviousSession(hint); } - // TODO: adapt tryEndPreviousSession(hint); to CrashExit.class if (HintUtils.hasType(hint, SessionStart.class)) { movePreviousSession(currentSessionFile, previousSessionFile); @@ -202,20 +203,20 @@ private boolean storeInternal(final @NotNull SentryEnvelope envelope, final @Not @SuppressWarnings("JavaUtilDate") private void tryEndPreviousSession(final @NotNull Hint hint) { final Object sdkHint = HintUtils.getSentrySdkHint(hint); - if (sdkHint instanceof AbnormalExit) { - final File previousSessionFile = getPreviousSessionFile(directory.getAbsolutePath()); + final File previousSessionFile = getPreviousSessionFile(directory.getAbsolutePath()); - if (previousSessionFile.exists()) { - options.getLogger().log(WARNING, "Previous session is not ended, we'd need to end it."); + if (previousSessionFile.exists()) { + options.getLogger().log(WARNING, "Previous session is not ended, we'd need to end it."); - try (final Reader reader = - new BufferedReader( - new InputStreamReader(new FileInputStream(previousSessionFile), UTF_8))) { - final Session session = serializer.getValue().deserialize(reader, Session.class); - if (session != null) { + try (final Reader reader = + new BufferedReader( + new InputStreamReader(new FileInputStream(previousSessionFile), UTF_8))) { + final Session session = serializer.getValue().deserialize(reader, Session.class); + if (session != null) { + Date timestamp = null; + if (sdkHint instanceof AbnormalExit) { final AbnormalExit abnormalHint = (AbnormalExit) sdkHint; final @Nullable Long abnormalExitTimestamp = abnormalHint.timestamp(); - Date timestamp = null; if (abnormalExitTimestamp != null) { timestamp = DateUtils.getDateTime(abnormalExitTimestamp); @@ -233,17 +234,33 @@ private void tryEndPreviousSession(final @NotNull Hint hint) { final String abnormalMechanism = abnormalHint.mechanism(); session.update(Session.State.Abnormal, null, true, abnormalMechanism); - // we have to use the actual timestamp of the Abnormal Exit here to mark the session - // as finished at the time it happened - session.end(timestamp); - writeSessionToDisk(previousSessionFile, session); + } else if (sdkHint instanceof NativeCrashExit) { + final NativeCrashExit nativeCrashHint = (NativeCrashExit) sdkHint; + final @NotNull Long nativeCrashExitTimestamp = nativeCrashHint.timestamp(); + + timestamp = DateUtils.getDateTime(nativeCrashExitTimestamp); + // sanity check if the native crash exit actually happened when the session was alive + final Date sessionStart = session.getStarted(); + if (sessionStart == null || timestamp.before(sessionStart)) { + options + .getLogger() + .log( + WARNING, + "Native crash exit happened before previous session start, not ending the session."); + return; + } + session.update(Session.State.Crashed, null, true, null); } - } catch (Throwable e) { - options.getLogger().log(SentryLevel.ERROR, "Error processing previous session.", e); + // we have to use the actual timestamp of the Abnormal or Crash Exit here to mark the + // session as finished at the time it happened + session.end(timestamp); + writeSessionToDisk(previousSessionFile, session); } - } else { - options.getLogger().log(DEBUG, "No previous session file to end."); + } catch (Throwable e) { + options.getLogger().log(SentryLevel.ERROR, "Error processing previous session.", e); } + } else { + options.getLogger().log(DEBUG, "No previous session file to end."); } } diff --git a/sentry/src/test/java/io/sentry/cache/EnvelopeCacheTest.kt b/sentry/src/test/java/io/sentry/cache/EnvelopeCacheTest.kt index 86fa209cfa2..1796fad708f 100644 --- a/sentry/src/test/java/io/sentry/cache/EnvelopeCacheTest.kt +++ b/sentry/src/test/java/io/sentry/cache/EnvelopeCacheTest.kt @@ -17,6 +17,7 @@ import io.sentry.UncaughtExceptionHandlerIntegration.UncaughtExceptionHint import io.sentry.cache.EnvelopeCache.PREFIX_CURRENT_SESSION_FILE import io.sentry.cache.EnvelopeCache.SUFFIX_SESSION_FILE import io.sentry.hints.AbnormalExit +import io.sentry.hints.NativeCrashExit import io.sentry.hints.SessionEndHint import io.sentry.hints.SessionStartHint import io.sentry.protocol.SentryId @@ -347,6 +348,53 @@ class EnvelopeCacheTest { assertEquals(null, updatedSession.abnormalMechanism) } + @Test + fun `NativeCrashExit hint marks previous session as crashed with crash timestamp`() { + val cache = fixture.getSUT() + + val previousSessionFile = EnvelopeCache.getPreviousSessionFile(fixture.options.cacheDirPath!!) + val session = createSession() + fixture.options.serializer.serialize(session, previousSessionFile.bufferedWriter()) + + val nativeCrashTimestamp = session.started!!.time + TimeUnit.HOURS.toMillis(3) + val envelope = SentryEnvelope.from(fixture.options.serializer, SentryEvent(), null) + val nativeCrashHint = NativeCrashExit { nativeCrashTimestamp } + val hints = HintUtils.createWithTypeCheckHint(nativeCrashHint) + cache.storeEnvelope(envelope, hints) + + val updatedSession = + fixture.options.serializer.deserialize( + previousSessionFile.bufferedReader(), + Session::class.java, + ) + assertEquals(State.Crashed, updatedSession!!.status) + assertEquals(nativeCrashTimestamp, updatedSession.timestamp!!.time) + } + + @Test + fun `when NativeCrashExit happened before previous session start, does not mark as crashed`() { + val cache = fixture.getSUT() + + val previousSessionFile = EnvelopeCache.getPreviousSessionFile(fixture.options.cacheDirPath!!) + val session = createSession() + val nativeCrashTimestamp = session.started!!.time - TimeUnit.HOURS.toMillis(3) + fixture.options.serializer.serialize(session, previousSessionFile.bufferedWriter()) + + val envelope = SentryEnvelope.from(fixture.options.serializer, SentryEvent(), null) + val nativeCrashHint = NativeCrashExit { nativeCrashTimestamp } + val hints = HintUtils.createWithTypeCheckHint(nativeCrashHint) + cache.storeEnvelope(envelope, hints) + + val updatedSession = + fixture.options.serializer.deserialize( + previousSessionFile.bufferedReader(), + Session::class.java, + ) + assertEquals(Ok, updatedSession!!.status) + assertTrue(nativeCrashTimestamp < updatedSession.started!!.time) + assertTrue(nativeCrashTimestamp < updatedSession.timestamp!!.time) + } + @Test fun `failing to store returns false`() { val serializer = mock() From f5de2020397072bb769d1f4b4e69d7377ca1058b Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Thu, 18 Dec 2025 12:58:08 +0100 Subject: [PATCH 33/35] move isBackgroundAnr() to the AnrHintEnricher since it is only used there. --- .../ApplicationExitInfoEventProcessor.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java index ba32c4976c2..ec60bd8f128 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/ApplicationExitInfoEventProcessor.java @@ -546,15 +546,6 @@ private void setDefaultPlatform(final @NotNull SentryBaseEvent event) { } } - // by default we assume that the ANR is foreground, unless abnormalMechanism is "anr_background" - private boolean isBackgroundAnr(final @NotNull Object hint) { - if (hint instanceof AbnormalExit) { - final String abnormalMechanism = ((AbnormalExit) hint).mechanism(); - return "anr_background".equals(abnormalMechanism); - } - return false; - } - private void mergeUser(final @NotNull SentryBaseEvent event) { @Nullable User user = event.getUser(); if (user == null) { @@ -687,6 +678,15 @@ public boolean supports(@NotNull Object hint) { return hint instanceof AbnormalExit; } + // by default we assume that the ANR is foreground, unless abnormalMechanism is "anr_background" + private boolean isBackgroundAnr(final @NotNull Object hint) { + if (hint instanceof AbnormalExit) { + final String abnormalMechanism = ((AbnormalExit) hint).mechanism(); + return "anr_background".equals(abnormalMechanism); + } + return false; + } + @Override public void applyPreEnrichment( @NotNull SentryEvent event, @NotNull Backfillable hint, @NotNull Object rawHint) { From c602ec950be31ae88260b6dded1691e953b0512a Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Thu, 18 Dec 2025 16:32:03 +0100 Subject: [PATCH 34/35] revert method-references back to lambdas --- .../sentry/android/core/cache/AndroidEnvelopeCache.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java b/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java index ddbfa0cc2dc..7eaec8333ad 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/cache/AndroidEnvelopeCache.java @@ -126,9 +126,9 @@ public static boolean hasStartupCrashMarker(final @NotNull SentryOptions options final File crashMarkerFile = new File(outboxPath, STARTUP_CRASH_MARKER_FILE); try { final boolean exists = - options.getRuntimeManager().runWithRelaxedPolicy(crashMarkerFile::exists); + options.getRuntimeManager().runWithRelaxedPolicy(() -> crashMarkerFile.exists()); if (exists) { - if (!options.getRuntimeManager().runWithRelaxedPolicy(crashMarkerFile::delete)) { + if (!options.getRuntimeManager().runWithRelaxedPolicy(() -> crashMarkerFile.delete())) { options .getLogger() .log( @@ -260,10 +260,10 @@ void handle( AnrV2Integration.AnrV2Hint.class, LAST_ANR_MARKER_LABEL, LAST_ANR_REPORT, - AnrV2Integration.AnrV2Hint::timestamp), + anrV2Hint -> anrV2Hint.timestamp()), new TimestampMarkerHandler<>( TombstoneIntegration.TombstoneHint.class, LAST_TOMBSTONE_MARKER_LABEL, LAST_TOMBSTONE_REPORT, - TombstoneIntegration.TombstoneHint::timestamp)); + tombstoneHint -> tombstoneHint.timestamp())); } From fd208cfe10f7eb978d4b0f79e4ec592823d10c3d Mon Sep 17 00:00:00 2001 From: Mischan Toosarani-Hausberger Date: Thu, 18 Dec 2025 16:54:55 +0100 Subject: [PATCH 35/35] clean up exception assignment in TombstoneParser --- .../android/core/internal/tombstone/TombstoneParser.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java index 3d60f762b83..67ecb1b74d0 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/internal/tombstone/TombstoneParser.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; public class TombstoneParser implements Closeable { @@ -56,10 +57,8 @@ public SentryEvent parse() throws IOException { event.setMessage(constructMessage(tombstone)); event.setDebugMeta(createDebugMeta(tombstone)); event.setExceptions(createException(tombstone)); - if (event.getExceptions() == null || event.getExceptions().isEmpty()) { - throw new RuntimeException("Failed to decode exception information from tombstone"); - } - event.setThreads(createThreads(tombstone, event.getExceptions().get(0))); + event.setThreads( + createThreads(tombstone, Objects.requireNonNull(event.getExceptions()).get(0))); return event; }