Update the default HTCondor and packer versions

community/modules/scripts/htcondor-install/README.md

@@ -134,7 +134,7 @@ No resources.
 
 | Name | Description | Type | Default | Required |
 | ---- | ----------- | ---- | ------- | :------: |
-| <a name="input_condor_version"></a> [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: "23.0.0","23.*")) | `string` | `"23.*"` | no |
+| <a name="input_condor_version"></a> [condor\_version](#input\_condor\_version) | Yum/DNF-compatible version string; leave unset to use latest 25.0 LTS release (examples: "25.0.0","25.*") | `string` | `"25.*"` | no |
 | <a name="input_enable_docker"></a> [enable\_docker](#input\_enable\_docker) | Install and enable docker daemon alongside HTCondor | `bool` | `true` | no |
 | <a name="input_http_proxy"></a> [http\_proxy](#input\_http\_proxy) | Set system default web (http and https) proxy for Windows HTCondor installation | `string` | `""` | no |
 | <a name="input_python_windows_installer_url"></a> [python\_windows\_installer\_url](#input\_python\_windows\_installer\_url) | URL of Python installer for Windows | `string` | `"https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe"` | no |

community/modules/scripts/htcondor-install/files/autoscaler.py

@@ -18,16 +18,20 @@
 # Script for resizing managed instance group (MIG) cluster size based
 # on the number of jobs in the Condor Queue.
 
+from absl import app
+from absl import flags
 from collections import OrderedDict
 from datetime import datetime
 from pprint import pprint
 from googleapiclient import discovery
 from oauth2client.client import GoogleCredentials
 
 import argparse
+import os
 import math
-import htcondor
-import classad
+import time
+import htcondor2 as htcondor
+import classad2 as classad
 
 parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 parser.add_argument("--p", required=True, help="Project id", type=str)
@@ -223,8 +227,8 @@ def scale(self):
 
         print(f"Last negotiation cycle occurred at: {datetime.fromtimestamp(last_negotiation_cycle_time)}")
         idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}")
-        idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query),
-                                    projection=job_attributes)
+        combined_query_str = f'JobStatus == 1 && QDate < {last_negotiation_cycle_time} && RequireId == "{self.instance_group_manager}"'
+        idle_job_ads = schedd.query(constraint=combined_query_str, projection=job_attributes)
 
         total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads)
         print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}")
@@ -263,19 +267,34 @@ def scale(self):
 
         # Find VMs that are idle (no dynamic slots created from partitionable
         # slots) in the MIG handled by this autoscaler
-        filter_idle_vms = classad.ExprTree("PartitionableSlot && NumDynamicSlots==0")
-        filter_claimed_vms = classad.ExprTree("PartitionableSlot && NumDynamicSlots>0")
+        filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0")
+        filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0")
         filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)")
+        combined_node_query = (
+                f'PartitionableSlot && NumDynamicSlots == 0 && '
+                f'regexp(".*/{self.instance_group_manager}$", CloudCreatedBy)'
+        )
+
+        filter_mig_str = f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)"
+        filter_claimed_vms_str = "PartitionableSlot && NumDynamicSlots>0"
+
         # A full list of Machine (StartD) ClassAd attributes can be found at
         # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html
-        idle_node_ads = coll.query(htcondor.AdTypes.Startd,
-            constraint=filter_idle_vms.and_(filter_mig),
-            projection=["Machine", "CloudZone"])
+
+        idle_node_ads = coll.query(
+            ad_type=htcondor.AdTypes.Startd,
+            constraint=combined_node_query,
+            projection=["Machine", "CloudZone"]
+        )
 
         NODENAME_ATTRIBUTE = "Machine"
-        claimed_node_ads = coll.query(htcondor.AdTypes.Startd,
-            constraint=filter_claimed_vms.and_(filter_mig),
-            projection=[NODENAME_ATTRIBUTE])
+        combined_constraint = classad.ExprTree(f"({filter_mig_str}) && ({filter_claimed_vms_str})")
+
+        claimed_node_ads = coll.query(
+            htcondor.AdType.Startd,  # Note: htcondor.AdTypes.Startd is replaced by htcondor.DaemonTypes.Startd or AdTypes if still supported, but DaemonTypes is standard in v2.
+            constraint=combined_constraint,
+            projection=[NODENAME_ATTRIBUTE]
+        )
         claimed_nodes = [ ad[NODENAME_ATTRIBUTE].split(".")[0] for ad in claimed_node_ads]
 
         # treat OrderedDict as a set by ignoring key values; this set will

community/modules/scripts/htcondor-install/files/install-htcondor.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # The instructions for installing HTCondor may change with time, although we
-# anticipate that they will stay fixed for the 23.0 releases. Find up-to-date
+# anticipate that they will stay fixed for the 25.0 releases. Find up-to-date
 # recommendations at:
 ## https://htcondor.readthedocs.io/en/latest/getting-htcondor/from-our-repositories.html
 
@@ -22,7 +22,7 @@
   hosts: all
   vars:
     enable_docker: true
-    htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-23.0-Key
+    htcondor_key: https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-25.0-Key
     docker_key: https://download.docker.com/linux/centos/gpg
   become: true
   module_defaults:
@@ -47,16 +47,16 @@
   - name: Enable HTCondor LTS Release repository
     ansible.builtin.yum_repository:
       name: htcondor-feature
-      description: HTCondor LTS Release (23.0)
+      description: HTCondor LTS Release (25.0)
       file: htcondor
-      baseurl: https://research.cs.wisc.edu/htcondor/repo/23.0/el$releasever/$basearch/release
+      baseurl: https://research.cs.wisc.edu/htcondor/repo/25.0/el$releasever/$basearch/release
       gpgkey: "{{ htcondor_key }}"
       gpgcheck: true
       repo_gpgcheck: true
       priority: "90"
   - name: Install HTCondor
     ansible.builtin.yum:
-      name: condor-{{ condor_version | default("23.*") | string }}
+      name: condor-{{ condor_version | default("25.*") | string }}
       state: present
   - name: Ensure token directory
     ansible.builtin.file:

community/modules/scripts/htcondor-install/variables.tf

@@ -21,16 +21,16 @@ variable "enable_docker" {
 }
 
 variable "condor_version" {
-  description = "Yum/DNF-compatible version string; leave unset to use latest 23.0 LTS release (examples: \"23.0.0\",\"23.*\"))"
+  description = "Yum/DNF-compatible version string; leave unset to use latest 25.0 LTS release (examples: \"25.0.0\",\"25.*\")"
   type        = string
-  default     = "23.*"
+  default     = "25.*"
 
   validation {
-    error_message = "var.condor_version must be set to \"23.*\" for latest 23.0 release or to a specific \"23.0.y\" release."
-    condition = var.condor_version == "23.*" || (
+    error_message = "var.condor_version must be set to \"25.*\" for latest 25.0 release or to a specific \"25.0.y\" release."
+    condition = var.condor_version == "25.*" || (
       length(split(".", var.condor_version)) == 3 && alltrue([
         for v in split(".", var.condor_version) : can(tonumber(v))
-      ]) && split(".", var.condor_version)[0] == "23"
+      ]) && split(".", var.condor_version)[0] == "25"
       && split(".", var.condor_version)[1] == "0"
     )
   }

tools/cloud-build/images/cluster-toolkit-dockerfile/Dockerfile

@@ -39,7 +39,7 @@ RUN wget -q "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terra
     rm terraform.zip
 
 # Install Packer
-ARG PACKER_VERSION=1.15.3
+ARG PACKER_VERSION=1.15.4
 RUN wget -q "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip" -O packer.zip && \
     unzip -o packer.zip && \
     mv packer /usr/local/bin/ && \

tools/update-dependencies.sh

@@ -18,7 +18,7 @@
 set -euo pipefail
 
 TERRAFORM_VERSION="1.12.2"
-PACKER_VERSION="1.15.3"
+PACKER_VERSION="1.15.4"
 OUTPUT_FILE="pkg/dependencies/checksums_generated.go"
 
 mkdir -p "$(dirname "$OUTPUT_FILE")"