Misc improvements

.github/workflows/tests.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   dotnet-bxes-unit-tests:
-    runs-on: windows-latest
+    runs-on: windows-2022
 
     steps:
       - uses: actions/checkout@v4
@@ -25,7 +25,7 @@ jobs:
         run: dotnet test ./bxes/src/csharp/Bxes.Tests
 
   rust-bxes-unit-tests:
-    runs-on: windows-latest
+    runs-on: windows-2022
 
     steps:
       - uses: actions/checkout@v4
@@ -37,7 +37,7 @@ jobs:
           RUSTFLAGS: ""
 
   dotnet-procfiler-unit-tests:
-    runs-on: windows-latest
+    runs-on: windows-2022
 
     steps:
       - uses: actions/checkout@v4

.run/unix/ProcfilerOnline.sh

@@ -4,11 +4,12 @@ export ProduceGcEvents=false
 export OnlineProcfilerSettings__KafkaSettings__TopicName=my-topic
 export OnlineProcfilerSettings__KafkaSettings__BootstrapServers=localhost:9092
 
+dotnet clean "$PM_IDE_ROOT"/All.sln
 dotnet build "$PM_IDE_ROOT"/Procfiler/src/dotnet/ProcfilerLoggerProvider/ -c Release
 dotnet build "$PM_IDE_ROOT"/Procfiler/test_data/source/WFWebApp/WFWebApp.csproj -c Release
 dotnet build "$PM_IDE_ROOT"/All.sln -c Release
 
-"$PM_IDE_ROOT"/Procfiler/src/dotnet/ProcfilerOnline/bin/Release/net9.0/ProcfilerOnline procfiler-online collect-online \
+"$PM_IDE_ROOT"/Procfiler/src/dotnet/ProcfilerOnline/bin/Release/net10.0/ProcfilerOnline procfiler-online collect-online \
  -dll-path "$PM_IDE_ROOT"/Procfiler/test_data/source/WFWebApp/bin/Release/net9.0/WFWebApp.dll \
  --target-methods-regex \
  WFWebApp \

.run/windows/Procfiler.cmd

@@ -1,6 +1,7 @@
+dotnet clean %PM_IDE_ROOT%/All.sln
 dotnet build %PM_IDE_ROOT%/All.sln
 
-%PM_IDE_ROOT%/Procfiler/src/dotnet/Procfiler/bin/Release/net9.0/Procfiler.exe procfiler^
+%PM_IDE_ROOT%/Procfiler/src/dotnet/Procfiler/bin/Release/net10.0/Procfiler.exe procfiler^
  collect-to-xes^
  -csproj^
  %PM_IDE_ROOT%\Procfiler\test_data\source\LOHAllocations\LOHAllocations.csproj^

.run/windows/ProcfilerOnline.cmd

@@ -1,16 +1,17 @@
 set ProduceEventsToKafka=true
 set ProduceBxesKafkaEvents=true
-set ProduceGcEvents=false
+set ProduceGcEvents=true
 set OnlineProcfilerSettings__KafkaSettings__TopicName=my-topic
 set OnlineProcfilerSettings__KafkaSettings__BootstrapServers=localhost:9092
 
+dotnet clean %PM_IDE_ROOT%/All.sln
 dotnet build %PM_IDE_ROOT%/All.sln -c Release
 dotnet build %PM_IDE_ROOT%/Procfiler/src/dotnet/ProcfilerLoggerProvider/ -c Release
 dotnet build %PM_IDE_ROOT%/Procfiler/test_data/source/WFWebApp -c Release
 
-%PM_IDE_ROOT%/Procfiler/src/dotnet/ProcfilerOnline/bin/Release/net9.0/ProcfilerOnline.exe procfiler-online collect-online^
- -dll-path %PM_IDE_ROOT%\Procfiler\test_data\source\WFWebApp\bin\Release\net9.0\WFWebApp.dll^
+%PM_IDE_ROOT%/Procfiler/src/dotnet/ProcfilerOnline/bin/Release/net10.0/ProcfilerOnline.exe procfiler-online collect-online^
+ -dll-path D:\work\DPN-Soundness-Verification\DPN.VerificationApp\bin\Debug\net10.0-windows\DPNVerifier.Desktop.dll^
  --target-methods-regex^
- WFWebApp^
+ DPN\.Soundness^
  --methods-filter-regex^
- WFWebApp^
+ DPN\.Soundness^

Ficus/examples/kafka_pipelines/activities_ecfg_pipeline.py

@@ -0,0 +1,88 @@
+from common import execute_pipeline, pipeline_with_default_cfg, PipelinePartInfo
+from ficus import *
+import os
+
+execute_pipeline(
+  'MySubscription',
+  'Pipeline',
+  [
+    pipeline_with_default_cfg([
+      RemainOnlyMethodStartEvents(),
+      SetMethodsDisplayName(),
+      ShortenAllocationType(),
+      PrepareSoftwareLog(time_attribute='QpcStamp'),
+      TerminateIfEmptyLog(),
+      TracesDiversityDiagramCanvas(),
+      AddStartEndArtificialEvents(),
+      DiscoverLoopsStrict(),
+      CreateLogFromActivitiesInstances(strategy=UndefinedActivityHandlingStrategy.InsertAllEvents),
+      ClearActivitiesRelatedStuff(),
+      DiscoverActivitiesForSeveralLevels(['.*'],
+                                         PatternsKind.MaximalRepeats,
+                                         activity_filter_kind=ActivityFilterKind.NoFilter),
+      DrawFullActivitiesDiagramCanvas(plot_legend=False, height_scale=10, width_scale=0.01),
+      ClusterizeActivitiesFromTracesDbscan(min_events_count_in_cluster=2,
+                                           tolerance=0.1,
+                                           activities_repr_source=ActivitiesRepresentationSource.EventClasses,
+                                           distance=Distance.Cosine,
+                                           activity_level=0,
+                                           view_params=(30, 60),
+                                           legend_cols=4,
+                                           visualization_method=DatasetVisualizationMethod.TSNE,
+                                           n_components=NComponents.Three),
+      DrawFullActivitiesDiagramCanvas(plot_legend=False, height_scale=10, width_scale=0.01),
+      CreateLogFromActivitiesInstances(strategy=UndefinedActivityHandlingStrategy.InsertAllEvents),
+      ClearActivitiesRelatedStuff(),
+      TracesDiversityDiagramCanvas(),
+      DiscoverECFG(root_sequence_kind=RootSequenceKind.FindBest,
+                   merge_sequences_of_events=False),
+      AnnotateGraphWithTime(TimeAnnotationKind.Mean),
+    ]),
+    PipelinePartInfo(
+      [
+        RemainEventsByRegex('GC/'),
+        FilterEventsByRegex('GC/RestartEEStart'),
+        FilterEventsByRegex('GC/RestartEEStop'),
+        FilterEventsByRegex('GC/SuspendEEStart'),
+        FilterEventsByRegex('GC/SuspendEEStop'),
+        FilterEventsByRegex('GC/Finaliz'),
+        FilterEventsByRegex('GC/SampledObject'),
+        FilterEventsByRegex('GC/SetGCHandle'),
+        FilterEventsByRegex('GC/BGCAllocWait'),
+        FilterEventsByRegex('GC/Pin'),
+        FilterEventsByRegex('GC/CreateSegment'),
+        FilterEventsByRegex('GC/Triggered'),
+        TerminateIfEmptyLog(),
+        DiscoverCases(start_regex='GC/Start', end_regex='GC/Stop', inline_inner_cases=True, pipeline=Pipeline(
+          DiscoverMultithreadedDfg(thread_attribute='NativeThreadId'),
+          ViewGraph(export_path=os.path.join(os.path.abspath(os.curdir), 'multithreaded.png')),
+          DiscoverDirectlyFollowsGraph(),
+          ViewGraph(export_path=os.path.join(os.path.abspath(os.curdir), 'default.png')),
+          AbstractMultithreadedEventsGroups(
+            thread_attribute='NativeThreadId',
+            time_attribute='QpcStamp',
+            n_components=NComponents.Two,
+            distance=Distance.Cosine,
+            tolerance=0.1,
+            view_params=(-50, 20),
+            put_noise_events_in_one_cluster=False,
+            min_events_count_in_cluster=2,
+            feature_count_kind=FeatureCountKind.Count,
+            visualization_method=DatasetVisualizationMethod.TSNE,
+            after_clusterization_pipeline=Pipeline(
+              AddStartEndArtificialEvents(),
+              TracesDiversityDiagramCanvas(),
+              DiscoverLoopsStrict(),
+              CreateLogFromActivitiesInstances(strategy=UndefinedActivityHandlingStrategy.InsertAllEvents),
+              DiscoverECFG(root_sequence_kind=RootSequenceKind.FindBest,
+                           merge_sequences_of_events=False),
+              AnnotateGraphWithTime(TimeAnnotationKind.Mean),
+            )
+          ),
+        ))
+      ],
+      os.path.join(os.path.abspath(os.curdir), 'gc_config.json'),
+      'GC pipeline'
+    )
+  ]
+)

Ficus/examples/kafka_pipelines/common.py

@@ -1,57 +1,80 @@
+from dataclasses import dataclass
+
 from ficus import *
 import os
 import time
 
+
 def env_or_default(env_name: str, default: str):
-    env = os.getenv(env_name)
-    return env if env is not None else default
-
-def execute_pipeline(sub_name: str, pipeline_name: str, pipeline_parts: list[PipelinePart], trace_filtering_pipeline = Pipeline()):
-    consumer_servers = env_or_default('CONSUMER_BOOTSTRAP_SERVERS', 'localhost:9092')
-    consumer_topic = env_or_default('CONSUMER_TOPIC', 'my-topic')
-    consumer_group = env_or_default('CONSUMER_GROUP_ID', 'xd')
-
-    kafka_consumer_metadata = KafkaPipelineMetadata(
-        topic_name=consumer_topic,
-        kafka_consumer_configuration={
-            'bootstrap.servers': consumer_servers,
-            'group.id': consumer_group,
-            'auto.offset.reset': 'earliest',
-            'message.max.bytes': '25728640',
-        }
-    )
-
-    producer_servers = env_or_default('PRODUCER_BOOTSTRAP_SERVERS', 'localhost:9092')
-    producer_topic = env_or_default('PRODUCER_TOPIC', 'ficus-topic')
-
-    kafka_producer_metadata = KafkaPipelineMetadata(
-        topic_name=producer_topic,
-        kafka_consumer_configuration={
-            'bootstrap.servers': producer_servers,
-            'message.max.bytes': '25728640',
-        }
-    )
-
-    ficus_backend = env_or_default('FICUS_BACKEND', 'localhost:8080')
-
-    subscription_id = create_kafka_subscription(sub_name, kafka_consumer_metadata, ficus_backend)
-    if subscription_id is None:
-        return
-
-    with open(os.path.join(os.path.abspath(os.curdir), 'software_data_config.json'), "r") as f:
-        software_data_config = f.read()
+  env = os.getenv(env_name)
+  return env if env is not None else default
+
+
+@dataclass
+class PipelinePartInfo:
+  parts: list[PipelinePart]
+  config_path: str
+  name: str
+
+
+def pipeline_with_default_cfg(pipeline_parts: list[PipelinePart]) -> PipelinePartInfo:
+  return PipelinePartInfo(
+    pipeline_parts,
+    os.path.join(os.path.abspath(os.curdir), 'software_data_config.json'),
+    'Pipeline'
+  )
+
+
+def execute_pipeline(sub_name: str,
+                     pipeline_name: str,
+                     pipelines: list[PipelinePartInfo],
+                     trace_filtering_pipeline=Pipeline()):
+  consumer_servers = env_or_default('CONSUMER_BOOTSTRAP_SERVERS', 'localhost:9092')
+  consumer_topic = env_or_default('CONSUMER_TOPIC', 'my-topic')
+  consumer_group = env_or_default('CONSUMER_GROUP_ID', 'xd')
+
+  kafka_consumer_metadata = KafkaPipelineMetadata(
+    topic_name=consumer_topic,
+    kafka_consumer_configuration={
+      'bootstrap.servers': consumer_servers,
+      'group.id': consumer_group,
+      'auto.offset.reset': 'earliest',
+      'message.max.bytes': '25728640',
+    }
+  )
+
+  producer_servers = env_or_default('PRODUCER_BOOTSTRAP_SERVERS', 'localhost:9092')
+  producer_topic = env_or_default('PRODUCER_TOPIC', 'ficus-topic')
+
+  kafka_producer_metadata = KafkaPipelineMetadata(
+    topic_name=producer_topic,
+    kafka_consumer_configuration={
+      'bootstrap.servers': producer_servers,
+      'message.max.bytes': '25728640',
+    }
+  )
+
+  ficus_backend = env_or_default('FICUS_BACKEND', 'localhost:8080')
+
+  subscription_id = create_kafka_subscription(sub_name, kafka_consumer_metadata, ficus_backend)
+  if subscription_id is None:
+    return
+
+  for p in pipelines:
+    with open(p.config_path, "r") as f:
+      software_data_config = f.read()
 
     KafkaPipeline(
-        pipeline_parts
-    ).execute(ficus_backend, 
-              subscription_id, 
-              pipeline_name, 
-              kafka_producer_metadata, 
+      p.parts
+    ).execute(ficus_backend,
+              subscription_id,
+              p.name,
+              kafka_producer_metadata,
               initial_context={
                 'software_data_extraction_config': JsonContextValue(software_data_config)
               },
-              streaming_configuration=create_queue_traces_configuration(3))
+              streaming_configuration=create_queue_traces_configuration(100))
 
-    if env_or_default('SLEEP', None) is not None:
-        while True:
-            time.sleep(10 ** 8)
+  if env_or_default('SLEEP', None) is not None:
+    while True:
+      time.sleep(10 ** 8)

Ficus/examples/kafka_pipelines/gc_config.json

@@ -0,0 +1,25 @@
+{
+  "pie_chart_extraction_configs": [
+    {
+      "event_class_regex": "GC\/SampledObject",
+      "info": {
+        "base": {
+          "name": "Object Allocations",
+          "units": "bytes"
+        },
+        "count_attr": "TotalSizeForTypeSample",
+        "grouping_attr": {
+          "single_attribute": {
+            "name": "TypeName",
+            "fallback_value": "UNRESOLVED"
+          }
+        }
+      }
+    }
+  ],
+  "simple_counter_configs": [],
+  "activities_duration_configs": [],
+  "raw_control_flow_regexes": [
+    "^((?!GC\\/SampledObject).)*$"
+  ]
+}

Ficus/examples/kafka_pipelines/gc_pipeline.py

@@ -1,30 +1,32 @@
-from common import execute_pipeline
+from common import execute_pipeline, pipeline_with_default_cfg
 from ficus import *
 
 execute_pipeline(
-    'MySubscription',
-    'GCPipeline',
-    [
-        RemainEventsByRegex('GC/'),
-        FilterEventsByRegex('GC/RestartEEStart'),
-        FilterEventsByRegex('GC/RestartEEStop'),
-        FilterEventsByRegex('GC/SuspendEEStart'),
-        FilterEventsByRegex('GC/SuspendEEStop'),
-        FilterEventsByRegex('GC/Finaliz'),
-        FilterEventsByRegex('GC/SampledObject'),
-        FilterEventsByRegex('GC/SetGCHandle'),
-        FilterEventsByRegex('GC/BGCAllocWait'),
-        FilterEventsByRegex('GC/Pin'),
-        FilterEventsByRegex('GC/CreateSegment'),
-        FilterEventsByRegex('GC/Triggered'),
-        FilterEventsByRegex('GC/Join'),
-        FilterEventsByRegex('Heap'),
-        FilterEventsByRegex('Bulk'),
-        FilterEventsByRegex('Mark'),
-        ShortenAllocationType(),
-        ShortenMethodNames(),
-        AddStartEndArtificialEvents(),
-        DiscoverECFG(root_sequence_kind=RootSequenceKind.FindBest,
-                     merge_sequences_of_events=True),
-    ]
+  'MySubscription',
+  'GCPipeline',
+  [
+    pipeline_with_default_cfg([
+      RemainEventsByRegex('GC/'),
+      FilterEventsByRegex('GC/RestartEEStart'),
+      FilterEventsByRegex('GC/RestartEEStop'),
+      FilterEventsByRegex('GC/SuspendEEStart'),
+      FilterEventsByRegex('GC/SuspendEEStop'),
+      FilterEventsByRegex('GC/Finaliz'),
+      FilterEventsByRegex('GC/SampledObject'),
+      FilterEventsByRegex('GC/SetGCHandle'),
+      FilterEventsByRegex('GC/BGCAllocWait'),
+      FilterEventsByRegex('GC/Pin'),
+      FilterEventsByRegex('GC/CreateSegment'),
+      FilterEventsByRegex('GC/Triggered'),
+      FilterEventsByRegex('GC/Join'),
+      FilterEventsByRegex('Heap'),
+      FilterEventsByRegex('Bulk'),
+      FilterEventsByRegex('Mark'),
+      ShortenAllocationType(),
+      ShortenMethodNames(),
+      AddStartEndArtificialEvents(),
+      DiscoverECFG(root_sequence_kind=RootSequenceKind.FindBest,
+                   merge_sequences_of_events=True),
+    ])
+  ]
 )