|
| 1 | +""" |
| 2 | +# Requirements: |
| 3 | + * Generate access token in your Github account, then create environment variable GITHUB_ACCESS_TOKEN. |
| 4 | + - e.g export GITHUB_ACCESS_TOKEN=1ns3rt-my-t0k3n-h3re. |
| 5 | + * Generate a service account key for your Google API credentials, then create environment variable GOOGLE_APPLICATION_CREDENTIALS. |
| 6 | + - e.g export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json. |
| 7 | +# Environment Variable/s: |
| 8 | + * IS_KALITE_RELEASE = Upload artifacts to the Google Cloud as a release candidate. |
| 9 | + * GITHUB_ACCESS_TOKEN = Personal access token used to authenticate in your Github account via API. |
| 10 | + * BUILDKITE_BUILD_NUMBER = Build identifier for each directory created. |
| 11 | + * BUILDKITE_PULL_REQUEST = Pull request issue or the value is false. |
| 12 | + * BUILDKITE_TAG = Tag identifier if this build was built from a tag. |
| 13 | + * BUILDKITE_COMMIT = Git commit hash that the build was made from. |
| 14 | + * GOOGLE_APPLICATION_CREDENTIALS = Your service account key. |
| 15 | +""" |
| 16 | + |
| 17 | +import logging |
| 18 | +import os |
| 19 | +import sys |
| 20 | + |
| 21 | +import requests |
| 22 | +from gcloud import storage |
| 23 | +from github3 import login |
| 24 | + |
| 25 | +logging.getLogger().setLevel(logging.INFO) |
| 26 | + |
| 27 | +ACCESS_TOKEN = os.getenv("GITHUB_ACCESS_TOKEN") |
| 28 | +REPO_OWNER = "learningequality" |
| 29 | +REPO_NAME = "ka-lite" |
| 30 | +ISSUE_ID = os.getenv("BUILDKITE_PULL_REQUEST") |
| 31 | +BUILD_ID = os.getenv("BUILDKITE_BUILD_NUMBER") |
| 32 | +TAG = os.getenv("BUILDKITE_TAG") |
| 33 | +COMMIT = os.getenv("BUILDKITE_COMMIT") |
| 34 | + |
| 35 | +RELEASE_DIR = 'release' |
| 36 | +PROJECT_PATH = os.path.join(os.getcwd()) |
| 37 | + |
| 38 | +# Python packages artifact location |
| 39 | +DIST_DIR = os.path.join(PROJECT_PATH, "dist") |
| 40 | +# Installer artifact location |
| 41 | +INSTALLER_DIR = os.path.join(PROJECT_PATH, "installer") |
| 42 | + |
| 43 | +headers = {'Authorization': 'token %s' % ACCESS_TOKEN} |
| 44 | + |
| 45 | +INSTALLER_CAT = "Installers" |
| 46 | +PYTHON_PKG_CAT = "Python Packages" |
| 47 | + |
| 48 | +# Manifest of files keyed by extension |
| 49 | + |
| 50 | +file_manifest = { |
| 51 | + 'exe': { |
| 52 | + 'extension': 'exe', |
| 53 | + 'description': 'Windows Installer', |
| 54 | + 'category': INSTALLER_CAT, |
| 55 | + 'content_type': 'application/x-ms-dos-executable', |
| 56 | + }, |
| 57 | + 'pex': { |
| 58 | + 'extension': 'pex', |
| 59 | + 'description': 'Pex file', |
| 60 | + 'category': PYTHON_PKG_CAT, |
| 61 | + 'content_type': 'application/octet-stream', |
| 62 | + }, |
| 63 | + 'whl': { |
| 64 | + 'extension': 'whl', |
| 65 | + 'description': 'Whl file', |
| 66 | + 'category': PYTHON_PKG_CAT, |
| 67 | + 'content_type': 'application/zip', |
| 68 | + }, |
| 69 | + 'gz': { |
| 70 | + 'extension': 'gz', |
| 71 | + 'description': 'Tar file', |
| 72 | + 'category': PYTHON_PKG_CAT, |
| 73 | + 'content_type': 'application/gzip', |
| 74 | + }, |
| 75 | +} |
| 76 | + |
| 77 | +file_order = [ |
| 78 | + 'exe', |
| 79 | + 'pex', |
| 80 | + 'whl', |
| 81 | + 'gz', |
| 82 | +] |
| 83 | + |
| 84 | +gh = login(token=ACCESS_TOKEN) |
| 85 | +repository = gh.repository(REPO_OWNER, REPO_NAME) |
| 86 | + |
| 87 | +def create_status_report_html(artifacts): |
| 88 | + """ |
| 89 | + Create html page to list build artifacts for linking from github status. |
| 90 | + """ |
| 91 | + html = "<html>\n<title>KA-Lite Buildkite Assets – Build #{build_id}</title>\n".format(build_id=BUILD_ID) |
| 92 | + html += "<body>\n<h1>Build Artifacts</h1>\n" |
| 93 | + current_heading = None |
| 94 | + |
| 95 | + for ext in file_order: |
| 96 | + artifacts_list = [] |
| 97 | + |
| 98 | + for artifact_dict in artifacts: |
| 99 | + if artifact_dict['extension'] == ext: |
| 100 | + artifacts_list.append(artifact_dict) |
| 101 | + |
| 102 | + for artifact in artifacts_list: |
| 103 | + if artifact['category'] != current_heading: |
| 104 | + current_heading = artifact['category'] |
| 105 | + html += "<h2>{heading}</h2>\n".format(heading=current_heading) |
| 106 | + html += "<p>{description}: <a href='{media_url}'>{name}</a></p>\n".format( |
| 107 | + **artifact) |
| 108 | + html += "</body>\n</html>" |
| 109 | + return html |
| 110 | + |
| 111 | +def create_github_status(report_url): |
| 112 | + """ |
| 113 | + Create a github status with a link to the report URL, |
| 114 | + only do this once buildkite has been successful, so only report success here. |
| 115 | + """ |
| 116 | + status = repository.create_status( |
| 117 | + COMMIT, |
| 118 | + "success", |
| 119 | + target_url=report_url, |
| 120 | + description="KA-Lite Buildkite assets", |
| 121 | + context="buildkite/kalite/assets" |
| 122 | + ) |
| 123 | + |
| 124 | + if status: |
| 125 | + logging.info("Successfully created GitHub status for commit {commit}.".format(commit=COMMIT)) |
| 126 | + else: |
| 127 | + logging.info("Error encountered. Now exiting!") |
| 128 | + sys.exit(1) |
| 129 | + |
| 130 | +def collect_local_artifacts(): |
| 131 | + """ |
| 132 | + Create a list of artifacts |
| 133 | + """ |
| 134 | + |
| 135 | + collected_artifacts = [] |
| 136 | + |
| 137 | + def create_artifact_data(artifact_dir): |
| 138 | + for artifact in os.listdir(artifact_dir): |
| 139 | + filename, file_extension = os.path.splitext(artifact) |
| 140 | + file_extension = file_extension[1:] # Remove leading '.' |
| 141 | + |
| 142 | + if file_extension in file_manifest: |
| 143 | + data = { |
| 144 | + 'name': artifact, |
| 145 | + 'file_location': "{artifact_dir}/{artifact}".format(artifact_dir=artifact_dir, artifact=artifact) |
| 146 | + } |
| 147 | + data.update(file_manifest[file_extension]) |
| 148 | + logging.info("Collecting file data: {data}".format(data=data)) |
| 149 | + collected_artifacts.append(data) |
| 150 | + |
| 151 | + create_artifact_data(DIST_DIR) |
| 152 | + create_artifact_data(INSTALLER_DIR) |
| 153 | + |
| 154 | + return collected_artifacts |
| 155 | + |
| 156 | +def upload_artifacts(): |
| 157 | + """ |
| 158 | + Upload the artifacts to the Google Cloud Storage |
| 159 | + Create a github status on the pull requester with the artifact media link. |
| 160 | + """ |
| 161 | + |
| 162 | + client = storage.Client() |
| 163 | + bucket = client.bucket("le-downloads") |
| 164 | + artifacts = collect_local_artifacts() |
| 165 | + is_release = os.getenv("IS_KALITE_RELEASE") |
| 166 | + |
| 167 | + for artifact in artifacts: |
| 168 | + logging.info("Uploading file {filename}".format(filename=artifact.get("name"))) |
| 169 | + |
| 170 | + if is_release: |
| 171 | + blob = bucket.blob("kalite/{release_dir}/{build_id}/{filename}".format( |
| 172 | + release_dir=RELEASE_DIR, |
| 173 | + build_id=BUILD_ID, |
| 174 | + filename=artifact.get("name") |
| 175 | + )) |
| 176 | + else: |
| 177 | + blob = bucket.blob("kalite/buildkite/build-{issue_id}/{build_id}/{filename}".format( |
| 178 | + issue_id=ISSUE_ID, |
| 179 | + build_id=BUILD_ID, |
| 180 | + filename=artifact.get("name") |
| 181 | + )) |
| 182 | + |
| 183 | + blob.upload_from_filename(filename=artifact.get("file_location")) |
| 184 | + blob.make_public() |
| 185 | + artifact.update({'media_url': blob.media_link}) |
| 186 | + |
| 187 | + html = create_status_report_html(artifacts) |
| 188 | + blob = bucket.blob("kalite/{release_dir}/{build_id}/report.html".format(release_dir=RELEASE_DIR, build_id=BUILD_ID)) |
| 189 | + blob.upload_from_string(html, content_type='text/html') |
| 190 | + blob.make_public() |
| 191 | + |
| 192 | + logging.info("Status Report link: {}".format(blob.public_url)) |
| 193 | + create_github_status(blob.public_url) |
| 194 | + |
| 195 | + if TAG: |
| 196 | + # Building from a tag, this is probably a release! |
| 197 | + get_release_asset_url = requests.get("https://api.github.com/repos/{owner}/{repo}/releases/tags/{tag}".format( |
| 198 | + owner=REPO_OWNER, |
| 199 | + repo=REPO_NAME, |
| 200 | + tag=TAG |
| 201 | + )) |
| 202 | + |
| 203 | + if get_release_asset_url.status_code == 200: |
| 204 | + release_id = get_release_asset_url.json()['id'] |
| 205 | + release_name = get_release_asset_url.json()['name'] |
| 206 | + release = repository.release(id=release_id) |
| 207 | + logging.info("Uploading build assets to GitHub Release: {release_name}".format(release_name=release_name)) |
| 208 | + |
| 209 | + for ext in file_order: |
| 210 | + artifact_list = [] |
| 211 | + for artifact_dict in artifacts: |
| 212 | + if artifact_dict['extension'] == ext: |
| 213 | + artifact_list.append(artifact_dict) |
| 214 | + |
| 215 | + for artifact in artifact_list: |
| 216 | + logging.info("Uploading release asset: {artifact_name}".format(artifact.get('name'))) |
| 217 | + # For some reason github3 does not let us set a label at initial upload |
| 218 | + asset = release.upload_asset( |
| 219 | + content_type=['content_type'], |
| 220 | + name=artifact['name'], |
| 221 | + asset=open(artifact['file_location'], 'rb') |
| 222 | + ) |
| 223 | + |
| 224 | + if asset: |
| 225 | + # So do it after the initial upload instead |
| 226 | + asset.edit(artifact['name'], label=artifact['description']) |
| 227 | + logging.info("Successfully uploaded release asset: {artifact}".format(artifact=artifact.get('name'))) |
| 228 | + else: |
| 229 | + logging.info("Error uploading release asset: {artifact}".format(artifact=artifact.get('name'))) |
| 230 | + |
| 231 | +def main(): |
| 232 | + upload_artifacts() |
| 233 | + |
| 234 | +if __name__ == "__main__": |
| 235 | + main() |
0 commit comments