Skip to content

Move format.py script and use retrying download #72993

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jakobbotsch merged 4 commits into from
Aug 4, 2022
Merged

Move format.py script and use retrying download #72993

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
48f6501
Move format.py script and use retrying download
jakobbotsch Jul 28, 2022
af30f83
Random jit change
jakobbotsch Jul 28, 2022
66697a0
Better jit change
jakobbotsch Jul 28, 2022
9677f78
Revert git changes
jakobbotsch Jul 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions eng/pipelines/coreclr/templates/format-job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
value: ${{ variable.value }}
- ${{ if ne(variable.group, '') }}:
- group: ${{ variable.group }}

- ${{ if eq(parameters.osGroup, 'windows') }}:
- name: PythonScript
value: 'py -3'
Expand All @@ -58,8 +58,8 @@ jobs:
version: '3.x'
includePreviewVersions: true
installationPath: $(Agent.ToolsDirectory)/dotnet
- script: $(PythonScript) $(Build.SourcesDirectory)/src/tests/Common/scripts/format.py -c $(Build.SourcesDirectory)/src/coreclr -o $(osGroup) -a $(archType)
displayName: Run format.py
- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitformat.py -c $(Build.SourcesDirectory)/src/coreclr -o $(osGroup) -a $(archType)
displayName: Run jitformat.py
- task: PublishBuildArtifacts@1
displayName: Publish format.patch
inputs:
Expand Down
86 changes: 41 additions & 45 deletions src/tests/Common/scripts/format.py → src/coreclr/scripts/jitformat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,26 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
## Licensed to the .NET Foundation under one or more agreements.
## The .NET Foundation licenses this file to you under the MIT license.
#
##
# Title :format.py
# Title :jitformat.py
#
################################################################################
# Script to install and run jit-format over jit source for all configurations.
################################################################################


import argparse
import jitutil
import logging
import os
import shutil
import subprocess
import sys
import tarfile
import tempfile
import zipfile
import subprocess
import shutil

class ChangeDir:
def __init__(self, dir):
Expand Down Expand Up @@ -46,13 +48,6 @@ def __enter__(self):
def __exit__(self, exc_type, exc_val, exc_tb):
os.chdir(self.cwd)

# Version specific imports

if sys.version_info.major < 3:
from urllib import urlretrieve
else:
from urllib.request import urlretrieve

def expandPath(path):
return os.path.abspath(os.path.expanduser(path))

Expand All @@ -62,14 +57,18 @@ def del_rw(action, name, exc):

def cleanup(jitUtilsPath, bootstrapPath):
if os.path.isdir(jitUtilsPath):
print("Deleting " + jitUtilsPath)
logging.info("Deleting " + jitUtilsPath)
shutil.rmtree(jitUtilsPath, onerror=del_rw)

if os.path.isfile(bootstrapPath):
print("Deleting " + bootstrapPath)
logging.info("Deleting " + bootstrapPath)
os.remove(bootstrapPath)

def main(argv):
logging.basicConfig(format="[%(asctime)s] %(message)s", datefmt="%H:%M:%S")
logger = logging.getLogger()
logger.setLevel(logging.INFO)

parser = argparse.ArgumentParser()
required = parser.add_argument_group('required arguments')
required.add_argument('-a', '--arch', type=str,
Expand All @@ -82,20 +81,20 @@ def main(argv):
args, unknown = parser.parse_known_args(argv)

if unknown:
print('Ignoring argument(s): ', ','.join(unknown))
logging.warn('Ignoring argument(s): {}'.format(','.join(unknown)))

if args.coreclr is None:
print('Specify --coreclr')
logging.error('Specify --coreclr')
return -1
if args.os is None:
print('Specify --os')
logging.error('Specify --os')
return -1
if args.arch is None:
print('Specify --arch')
logging.error('Specify --arch')
return -1

if not os.path.isdir(expandPath(args.coreclr)):
print('Bad path to coreclr')
logging.error('Bad path to coreclr')
return -1

coreclr = args.coreclr.replace('/', os.sep)
Expand All @@ -117,7 +116,7 @@ def main(argv):
proc = subprocess.Popen(formattingDownloadScriptCommand)

if proc.wait() != 0:
print("Formatting tool download failed")
logging.error("Formatting tool download failed")
return -1

my_env["PATH"] = os.path.join(repoRoot, "artifacts", "tools") + os.pathsep + my_env["PATH"]
Expand All @@ -142,17 +141,14 @@ def main(argv):

assert len(os.listdir(os.path.dirname(bootstrapPath))) == 0

print('Downloading', bootstrapUrl, 'to', bootstrapPath)
urlretrieve(bootstrapUrl, bootstrapPath)

if not os.path.isfile(bootstrapPath):
print("Did not download bootstrap!")
if not jitutil.download_one_url(bootstrapUrl, bootstrapPath):
logging.error("Did not download bootstrap!")
return -1

if platform == 'windows':
# Need to ensure we have Windows line endings on the downloaded script file,
# which is downloaded with Unix line endings.
print('Convert', bootstrapPath, 'to Windows line endings')
logging.info('Convert {} to Windows line endings'.format(bootstrapPath))

content = None
with open(bootstrapPath, 'rb') as open_file:
Expand All @@ -166,22 +162,22 @@ def main(argv):
# On *nix platforms, we need to make the bootstrap file executable

if platform == 'Linux' or platform == 'OSX':
print("Making bootstrap executable")
logging.info("Making bootstrap executable")
os.chmod(bootstrapPath, 0o751)

# Run bootstrap
if platform == 'Linux' or platform == 'OSX':
print('Running:', 'bash', bootstrapPath)
logging.info('Running: bash {}'.format(bootstrapPath))
proc = subprocess.Popen(['bash', bootstrapPath], env=my_env)
output,error = proc.communicate()
elif platform == 'windows':
print('Running:', bootstrapPath)
logging.info('Running: {}'.format(bootstrapPath))
proc = subprocess.Popen([bootstrapPath], env=my_env)
output,error = proc.communicate()

if proc.returncode != 0:
cleanup('', bootstrapPath)
print("Bootstrap failed")
logging.error("Bootstrap failed")
return -1

# Run jit-format
Expand All @@ -191,7 +187,7 @@ def main(argv):
my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"]

if not os.path.isdir(jitutilsBin):
print("Jitutils not built!")
logging.error("Jitutils not built!")
return -1

jitformat = jitutilsBin
Expand All @@ -208,7 +204,7 @@ def main(argv):
for build in builds:
for project in projects:
command = jitformat + " -a " + arch + " -b " + build + " -o " + platform + " -c " + coreclr + " --verbose --projects " + project
print('Running:', command)
logging.info('Running: {}'.format(command))
proc = subprocess.Popen([jitformat, "-a", arch, "-b", build, "-o", platform, "-c", coreclr, "--verbose", "--projects", project], env=my_env)
output,error = proc.communicate()
errorcode = proc.returncode
Expand All @@ -234,7 +230,7 @@ def main(argv):

if returncode != 0:
# Create a patch file
print("Creating patch file " + patchFilePath)
logging.info("Creating patch file {}".format(patchFilePath))
jitSrcPath = os.path.join(coreclr, "jit")
patchFile = open(patchFilePath, "w")
proc = subprocess.Popen(["git", "diff", "--patch", "-U20", "--", jitSrcPath], env=my_env, stdout=patchFile)
Expand All @@ -243,19 +239,19 @@ def main(argv):
cleanup(jitUtilsPath, bootstrapPath)

if returncode != 0:
print("There were errors in formatting. Please run jit-format locally with: \n")
print(errorMessage)
print("\nOr download and apply generated patch:")
print("1. From the GitHub 'Checks' page on the Pull Request, with the failing Formatting")
print(" job selected (e.g., 'Formatting Linux x64'), click the 'View more details on")
print(" Azure Pipelines' link.")
print("2. Select the '1 artifact produced' at the end of the log.")
print("3. Artifacts are located in alphabetical order, target artifact name is")
print(" 'format.<OS>.<architecture>.patch.'. Find appropriate format patch artifact.")
print("4. On the right side of the artifact there is a 'More actions' menu shown by a")
print(" vertical three-dot symbol. Click on it and select 'Download artifacts' option.")
print("5. Unzip the patch file.")
print("6. git apply format.patch")
logging.info("There were errors in formatting. Please run jit-format locally with: \n")
logging.info(errorMessage)
logging.info("\nOr download and apply generated patch:")
logging.info("1. From the GitHub 'Checks' page on the Pull Request, with the failing Formatting")
logging.info(" job selected (e.g., 'Formatting Linux x64'), click the 'View more details on")
logging.info(" Azure Pipelines' link.")
logging.info("2. Select the '1 artifact produced' at the end of the log.")
logging.info("3. Artifacts are located in alphabetical order, target artifact name is")
logging.info(" 'format.<OS>.<architecture>.patch.'. Find appropriate format patch artifact.")
logging.info("4. On the right side of the artifact there is a 'More actions' menu shown by a")
logging.info(" vertical three-dot symbol. Click on it and select 'Download artifacts' option.")
logging.info("5. Unzip the patch file.")
logging.info("6. git apply format.patch")

if (returncode != 0) and (os.environ.get("TF_BUILD") == "True"):
print("##vso[task.logissue type=error](NETCORE_ENGINEERING_TELEMETRY=Build) Format job found errors, please apply the format patch.")
Expand Down