代码拉取完成，页面将自动刷新
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Gclient-specific SCM-specific operations."""
import collections
import contextlib
import errno
import json
import logging
import os
import platform
import posixpath
import re
import shutil
import sys
import tempfile
import threading
import traceback
import gclient_utils
import gerrit_util
import git_cache
import scm
import subprocess2
# TODO: Should fix these warnings.
# pylint: disable=line-too-long
class NoUsableRevError(gclient_utils.Error):
"""Raised if requested revision isn't found in checkout."""
class DiffFiltererWrapper(object):
"""Simple base class which tracks which file is being diffed and
replaces instances of its file name in the original and
working copy lines of the git diff output."""
index_string = None
original_prefix = "--- "
working_prefix = "+++ "
def __init__(self, relpath, print_func):
# Note that we always use '/' as the path separator to be
# consistent with cygwin-style output on Windows
self._relpath = relpath.replace("\\", "/")
self._current_file = None
self._print_func = print_func
def SetCurrentFile(self, current_file):
self._current_file = current_file
@property
def _replacement_file(self):
return posixpath.join(self._relpath, self._current_file)
def _Replace(self, line):
return line.replace(self._current_file, self._replacement_file)
def Filter(self, line):
if (line.startswith(self.index_string)):
self.SetCurrentFile(line[len(self.index_string):])
line = self._Replace(line)
else:
if (line.startswith(self.original_prefix)
or line.startswith(self.working_prefix)):
line = self._Replace(line)
self._print_func(line)
class GitDiffFilterer(DiffFiltererWrapper):
index_string = "diff --git "
def SetCurrentFile(self, current_file):
# Get filename by parsing "a/<filename> b/<filename>"
self._current_file = current_file[:(len(current_file) / 2)][2:]
def _Replace(self, line):
return re.sub("[a|b]/" + self._current_file, self._replacement_file,
line)
# SCMWrapper base class
class SCMWrapper(object):
"""Add necessary glue between all the supported SCM.
This is the abstraction layer to bind to different SCM.
"""
def __init__(self,
url=None,
root_dir=None,
relpath=None,
out_fh=None,
out_cb=None,
print_outbuf=False):
self.url = url
self._root_dir = root_dir
if self._root_dir:
self._root_dir = self._root_dir.replace('/', os.sep)
self.relpath = relpath
if self.relpath:
self.relpath = self.relpath.replace('/', os.sep)
if self.relpath and self._root_dir:
self.checkout_path = os.path.join(self._root_dir, self.relpath)
if out_fh is None:
out_fh = sys.stdout
self.out_fh = out_fh
self.out_cb = out_cb
self.print_outbuf = print_outbuf
def Print(self, *args, **kwargs):
kwargs.setdefault('file', self.out_fh)
if kwargs.pop('timestamp', True):
self.out_fh.write('[%s] ' % gclient_utils.Elapsed())
print(*args, **kwargs)
def RunCommand(self, command, options, args, file_list=None):
commands = [
'update', 'updatesingle', 'revert', 'revinfo', 'status', 'diff',
'pack', 'runhooks'
]
if not command in commands:
raise gclient_utils.Error('Unknown command %s' % command)
if not command in dir(self):
raise gclient_utils.Error(
'Command %s not implemented in %s wrapper' %
(command, self.__class__.__name__))
return getattr(self, command)(options, args, file_list)
@staticmethod
def _get_first_remote_url(checkout_path):
log = scm.GIT.Capture(
['config', '--local', '--get-regexp', r'remote.*.url'],
cwd=checkout_path)
# Get the second token of the first line of the log.
return log.splitlines()[0].split(' ', 1)[1]
def GetCacheMirror(self):
if getattr(self, 'cache_dir', None):
url, _ = gclient_utils.SplitUrlRevision(self.url)
return git_cache.Mirror(url)
return None
def GetActualRemoteURL(self, options):
"""Attempt to determine the remote URL for this SCMWrapper."""
# Git
if os.path.exists(os.path.join(self.checkout_path, '.git')):
actual_remote_url = self._get_first_remote_url(self.checkout_path)
mirror = self.GetCacheMirror()
# If the cache is used, obtain the actual remote URL from there.
if (mirror and mirror.exists() and mirror.mirror_path.replace(
'\\', '/') == actual_remote_url.replace('\\', '/')):
actual_remote_url = self._get_first_remote_url(
mirror.mirror_path)
return actual_remote_url
return None
def DoesRemoteURLMatch(self, options):
"""Determine whether the remote URL of this checkout is the expected URL."""
if not os.path.exists(self.checkout_path):
# A checkout which doesn't exist can't be broken.
return True
actual_remote_url = self.GetActualRemoteURL(options)
if actual_remote_url:
return (gclient_utils.SplitUrlRevision(actual_remote_url)[0].rstrip(
'/') == gclient_utils.SplitUrlRevision(self.url)[0].rstrip('/'))
# This may occur if the self.checkout_path exists but does not contain a
# valid git checkout.
return False
def _DeleteOrMove(self, force):
"""Delete the checkout directory or move it out of the way.
Args:
force: bool; if True, delete the directory. Otherwise, just move it.
"""
if force and os.environ.get('CHROME_HEADLESS') == '1':
self.Print('_____ Conflicting directory found in %s. Removing.' %
self.checkout_path)
gclient_utils.AddWarning('Conflicting directory %s deleted.' %
self.checkout_path)
gclient_utils.rmtree(self.checkout_path)
else:
bad_scm_dir = os.path.join(self._root_dir, '_bad_scm',
os.path.dirname(self.relpath))
try:
os.makedirs(bad_scm_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
dest_path = tempfile.mkdtemp(prefix=os.path.basename(self.relpath),
dir=bad_scm_dir)
self.Print(
'_____ Conflicting directory found in %s. Moving to %s.' %
(self.checkout_path, dest_path))
gclient_utils.AddWarning('Conflicting directory %s moved to %s.' %
(self.checkout_path, dest_path))
shutil.move(self.checkout_path, dest_path)
class GitWrapper(SCMWrapper):
"""Wrapper for Git"""
name = 'git'
remote = 'origin'
_is_env_cog = None
@staticmethod
def _IsCog():
"""Returns true if the env is cog"""
if not GitWrapper._is_env_cog:
GitWrapper._is_env_cog = any(
os.getcwd().startswith(x)
for x in ['/google/cog/cloud', '/google/src/cloud'])
return GitWrapper._is_env_cog
@property
def cache_dir(self):
try:
return git_cache.Mirror.GetCachePath()
except RuntimeError:
return None
def __init__(self, url=None, *args, **kwargs):
"""Removes 'git+' fake prefix from git URL."""
if url and (url.startswith('git+http://')
or url.startswith('git+https://')):
url = url[4:]
SCMWrapper.__init__(self, url, *args, **kwargs)
filter_kwargs = {'time_throttle': 1, 'out_fh': self.out_fh}
if self.out_cb:
filter_kwargs['predicate'] = self.out_cb
self.filter = gclient_utils.GitFilter(**filter_kwargs)
self._running_under_rosetta = None
def GetCheckoutRoot(self):
return scm.GIT.GetCheckoutRoot(self.checkout_path)
def GetRevisionDate(self, _revision):
"""Returns the given revision's date in ISO-8601 format (which contains the
time zone)."""
# TODO(floitsch): get the time-stamp of the given revision and not just
# the time-stamp of the currently checked out revision.
return self._Capture(['log', '-n', '1', '--format=%ai'])
def _GetDiffFilenames(self, base):
"""Returns the names of files modified since base."""
return self._Capture(
# Filter to remove base if it is None.
list(
filter(
bool,
['-c', 'core.quotePath=false', 'diff', '--name-only', base])
)).split()
def diff(self, options, _args, _file_list):
_, revision = gclient_utils.SplitUrlRevision(self.url)
if not revision:
revision = 'refs/remotes/%s/main' % self.remote
self._Run(['-c', 'core.quotePath=false', 'diff', revision], options)
def pack(self, _options, _args, _file_list):
"""Generates a patch file which can be applied to the root of the
repository.
The patch file is generated from a diff of the merge base of HEAD and
its upstream branch.
"""
try:
merge_base = [self._Capture(['merge-base', 'HEAD', self.remote])]
except subprocess2.CalledProcessError:
merge_base = []
gclient_utils.CheckCallAndFilter(['git', 'diff'] + merge_base,
cwd=self.checkout_path,
filter_fn=GitDiffFilterer(
self.relpath,
print_func=self.Print).Filter)
def _Scrub(self, target, options):
"""Scrubs out all changes in the local repo, back to the state of target."""
quiet = []
if not options.verbose:
quiet = ['--quiet']
self._Run(['reset', '--hard', target] + quiet, options)
if options.force and options.delete_unversioned_trees:
# where `target` is a commit that contains both upper and lower case
# versions of the same file on a case insensitive filesystem, we are
# actually in a broken state here. The index will have both 'a' and
# 'A', but only one of them will exist on the disk. To progress, we
# delete everything that status thinks is modified.
output = self._Capture(
['-c', 'core.quotePath=false', 'status', '--porcelain'],
strip=False)
for line in output.splitlines():
# --porcelain (v1) looks like:
# XY filename
try:
filename = line[3:]
self.Print('_____ Deleting residual after reset: %r.' %
filename)
gclient_utils.rm_file_or_tree(
os.path.join(self.checkout_path, filename))
except OSError:
pass
def _FetchAndReset(self, revision, file_list, options):
"""Equivalent to git fetch; git reset."""
self._SetFetchConfig(options)
self._Fetch(options, prune=True, quiet=options.verbose)
self._Scrub(revision, options)
if file_list is not None:
files = self._Capture(['-c', 'core.quotePath=false',
'ls-files']).splitlines()
file_list.extend(
[os.path.join(self.checkout_path, f) for f in files])
def _DisableHooks(self):
hook_dir = os.path.join(self.checkout_path, '.git', 'hooks')
if not os.path.isdir(hook_dir):
return
for f in os.listdir(hook_dir):
if not f.endswith('.sample') and not f.endswith('.disabled'):
disabled_hook_path = os.path.join(hook_dir, f + '.disabled')
if os.path.exists(disabled_hook_path):
os.remove(disabled_hook_path)
os.rename(os.path.join(hook_dir, f), disabled_hook_path)
def _maybe_break_locks(self, options):
"""This removes all .lock files from this repo's .git directory, if the
user passed the --break_repo_locks command line flag.
In particular, this will cleanup index.lock files, as well as ref lock
files.
"""
if options.break_repo_locks:
git_dir = os.path.join(self.checkout_path, '.git')
for path, _, filenames in os.walk(git_dir):
for filename in filenames:
if filename.endswith('.lock'):
to_break = os.path.join(path, filename)
self.Print('breaking lock: %s' % (to_break, ))
try:
os.remove(to_break)
except OSError as ex:
self.Print('FAILED to break lock: %s: %s' %
(to_break, ex))
raise
def _download_topics(self, patch_rev, googlesource_url):
"""This method returns new patch_revs to process that have the same topic.
It does the following:
1. Finds the topic of the Gerrit change specified in the patch_rev.
2. Find all changes with that topic.
3. Append patch_rev of the changes with the same topic to the patch_revs
to process.
4. Returns the new patch_revs to process.
"""
patch_revs_to_process = []
# Parse the patch_rev to extract the CL and patchset.
patch_rev_tokens = patch_rev.split('/')
change = patch_rev_tokens[-2]
# Parse the googlesource_url.
tokens = re.search('//(.+).googlesource.com/(.+?)(?:\.git)?$',
googlesource_url)
if not tokens or len(tokens.groups()) != 2:
# googlesource_url is not in the expected format.
return patch_revs_to_process
# parse the gerrit host and repo out of googlesource_url.
host, repo = tokens.groups()[:2]
gerrit_host_url = '%s-review.googlesource.com' % host
# 1. Find the topic of the Gerrit change specified in the patch_rev.
change_object = gerrit_util.GetChange(gerrit_host_url, change)
topic = change_object.get('topic')
if not topic:
# This change has no topic set.
return patch_revs_to_process
# 2. Find all changes with that topic.
changes_with_same_topic = gerrit_util.QueryChanges(
gerrit_host_url, [('topic', topic), ('status', 'open'),
('repo', repo)],
o_params=['ALL_REVISIONS'])
for c in changes_with_same_topic:
if str(c['_number']) == change:
# This change is already in the patch_rev.
continue
self.Print('Found CL %d with the topic name %s' %
(c['_number'], topic))
# 3. Append patch_rev of the changes with the same topic to the
# patch_revs to process.
curr_rev = c['current_revision']
new_patch_rev = c['revisions'][curr_rev]['ref']
patch_revs_to_process.append(new_patch_rev)
# 4. Return the new patch_revs to process.
return patch_revs_to_process
def _ref_to_remote_ref(self, target_rev):
"""Helper function for scm.GIT.RefToRemoteRef with error checking.
Joins the results of scm.GIT.RefToRemoteRef into a string, but raises a
comprehensible error if RefToRemoteRef fails.
Args:
target_rev: a ref somewhere under refs/.
"""
tmp_ref = scm.GIT.RefToRemoteRef(target_rev, self.remote)
if not tmp_ref:
raise gclient_utils.Error(
'Failed to turn target revision %r in repo %r into remote ref' %
(target_rev, self.checkout_path))
return ''.join(tmp_ref)
def apply_patch_ref(self, patch_repo, patch_rev, target_rev, options,
file_list):
# type: (str, str, str, optparse.Values, Collection[str]) -> str
"""Apply a patch on top of the revision we're synced at.
The patch ref is given by |patch_repo|@|patch_rev|.
|target_rev| is usually the branch that the |patch_rev| was uploaded against
(e.g. 'refs/heads/main'), but this is not required.
We cherry-pick all commits reachable from |patch_rev| on top of the curret
HEAD, excluding those reachable from |target_rev|
(i.e. git cherry-pick target_rev..patch_rev).
Graphically, it looks like this:
... -> o -> [possibly already landed commits] -> target_rev
\
-> [possibly not yet landed dependent CLs] -> patch_rev
The final checkout state is then:
... -> HEAD -> [possibly not yet landed dependent CLs] -> patch_rev
After application, if |options.reset_patch_ref| is specified, we soft reset
the cherry-picked changes, keeping them in git index only.
Args:
patch_repo: The patch origin.
e.g. 'https://foo.googlesource.com/bar'
patch_rev: The revision to patch.
e.g. 'refs/changes/1234/34/1'.
target_rev: The revision to use when finding the merge base.
Typically, the branch that the patch was uploaded against.
e.g. 'refs/heads/main' or 'refs/heads/infra/config'.
options: The options passed to gclient.
file_list: A list where modified files will be appended.
"""
# Abort any cherry-picks in progress.
try:
self._Capture(['cherry-pick', '--abort'])
except subprocess2.CalledProcessError:
pass
base_rev = self.revinfo(None, None, None)
if not target_rev:
raise gclient_utils.Error(
'A target revision for the patch must be given')
if target_rev.startswith(('refs/heads/', 'refs/branch-heads')):
# If |target_rev| is in refs/heads/** or refs/branch-heads/**, try
# first to find the corresponding remote ref for it, since
# |target_rev| might point to a local ref which is not up to date
# with the corresponding remote ref.
remote_ref = self._ref_to_remote_ref(target_rev)
self.Print('Trying the corresponding remote ref for %r: %r\n' %
(target_rev, remote_ref))
if scm.GIT.IsValidRevision(self.checkout_path, remote_ref):
# refs/remotes may need to be updated to cleanly cherry-pick
# changes. See https://crbug.com/1255178.
self._Capture(['fetch', '--no-tags', self.remote, target_rev])
target_rev = remote_ref
elif not scm.GIT.IsValidRevision(self.checkout_path, target_rev):
# Fetch |target_rev| if it's not already available.
url, _ = gclient_utils.SplitUrlRevision(self.url)
mirror = self._GetMirror(url, options, target_rev, target_rev)
if mirror:
rev_type = 'branch' if target_rev.startswith(
'refs/') else 'hash'
self._UpdateMirrorIfNotContains(mirror, options, rev_type,
target_rev)
self._Fetch(options, refspec=target_rev)
patch_revs_to_process = [patch_rev]
if hasattr(options, 'download_topics') and options.download_topics:
patch_revs_to_process_from_topics = self._download_topics(
patch_rev, self.url)
patch_revs_to_process.extend(patch_revs_to_process_from_topics)
self._Capture(['reset', '--hard'])
for pr in patch_revs_to_process:
self.Print('===Applying patch===')
self.Print('Revision to patch is %r @ %r.' % (patch_repo, pr))
self.Print('Current dir is %r' % self.checkout_path)
self._Capture(['fetch', '--no-tags', patch_repo, pr])
pr = self._Capture(['rev-parse', 'FETCH_HEAD'])
if not options.rebase_patch_ref:
self._Capture(['checkout', pr])
# Adjust base_rev to be the first parent of our checked out
# patch ref; This will allow us to correctly extend `file_list`,
# and will show the correct file-list to programs which do `git
# diff --cached` expecting to see the patch diff.
base_rev = self._Capture(['rev-parse', pr + '~'])
else:
self.Print('Will cherrypick %r .. %r on top of %r.' %
(target_rev, pr, base_rev))
try:
if scm.GIT.IsAncestor(pr,
target_rev,
cwd=self.checkout_path):
if len(patch_revs_to_process) > 1:
# If there are multiple patch_revs_to_process then
# we do not want want to invalidate a previous patch
# so throw an error.
raise gclient_utils.Error(
'patch_rev %s is an ancestor of target_rev %s. This '
'situation is unsupported when we need to apply multiple '
'patch_revs: %s' %
(pr, target_rev, patch_revs_to_process))
# If |patch_rev| is an ancestor of |target_rev|, check
# it out.
self._Capture(['checkout', pr])
else:
# If a change was uploaded on top of another change,
# which has already landed, one of the commits in the
# cherry-pick range will be redundant, since it has
# already landed and its changes incorporated in the
# tree. We pass '--keep-redundant-commits' to ignore
# those changes.
self._Capture([
'cherry-pick', target_rev + '..' + pr,
'--keep-redundant-commits'
])
except subprocess2.CalledProcessError as e:
self.Print('Failed to apply patch.')
self.Print('Revision to patch was %r @ %r.' %
(patch_repo, pr))
self.Print('Tried to cherrypick %r .. %r on top of %r.' %
(target_rev, pr, base_rev))
self.Print('Current dir is %r' % self.checkout_path)
self.Print('git returned non-zero exit status %s:\n%s' %
(e.returncode, e.stderr.decode('utf-8')))
# Print the current status so that developers know what
# changes caused the patch failure, since git cherry-pick
# doesn't show that information.
self.Print(self._Capture(['status']))
try:
self._Capture(['cherry-pick', '--abort'])
except subprocess2.CalledProcessError:
pass
raise
if file_list is not None:
file_list.extend(self._GetDiffFilenames(base_rev))
latest_commit = self.revinfo(None, None, None)
if options.reset_patch_ref:
self._Capture(['reset', '--soft', base_rev])
return latest_commit
def check_diff(self, previous_commit, files=None):
# type: (str, Optional[List[str]]) -> bool
"""Check if a diff exists between the current commit and `previous_commit`.
Returns True if there were diffs or if an error was encountered.
"""
cmd = ['diff', previous_commit, '--quiet']
if files:
cmd += ['--'] + files
try:
self._Capture(cmd)
return False
except subprocess2.CalledProcessError as e:
# git diff --quiet exits with 1 if there were diffs.
if e.returncode != 1:
self.Print('git returned non-zero exit status %s:\n%s' %
(e.returncode, e.stderr.decode('utf-8')))
return True
def set_config(f):
def wrapper(*args):
return_val = f(*args)
if os.path.exists(os.path.join(args[0].checkout_path, '.git')):
# If diff.ignoreSubmodules is not already set, set it to `all`.
config = subprocess2.capture(['git', 'config', '-l'],
cwd=args[0].checkout_path).decode(
'utf-8').strip().splitlines()
if 'diff.ignoresubmodules=dirty' not in config:
subprocess2.capture(
['git', 'config', 'diff.ignoreSubmodules', 'dirty'],
cwd=args[0].checkout_path)
if 'fetch.recursesubmodules=off' not in config:
subprocess2.capture(
['git', 'config', 'fetch.recurseSubmodules', 'off'],
cwd=args[0].checkout_path)
if 'push.recursesubmodules=off' not in config:
# The default is off, but if user sets submodules.recurse to
# on, this becomes on too. We never want to push submodules
# for gclient managed repositories. Push, even if a no-op,
# will increase `git cl upload` latency.
subprocess2.capture(
['git', 'config', 'push.recurseSubmodules', 'off'],
cwd=args[0].checkout_path)
return return_val
return wrapper
@set_config
def update(self, options, args, file_list):
"""Runs git to update or transparently checkout the working copy.
All updated files will be appended to file_list.
Raises:
Error: if can't get URL for relative path.
"""
if args:
raise gclient_utils.Error("Unsupported argument(s): %s" %
",".join(args))
self._CheckMinVersion("1.6.6")
url, deps_revision = gclient_utils.SplitUrlRevision(self.url)
revision = deps_revision
managed = True
if options.revision:
# Override the revision number.
revision = str(options.revision)
if revision == 'unmanaged':
# Check again for a revision in case an initial ref was specified
# in the url, for example bla.git@refs/heads/custombranch
revision = deps_revision
managed = False
if not revision:
# If a dependency is not pinned, track the default remote branch.
revision = scm.GIT.GetRemoteHeadRef(self.checkout_path, self.url,
self.remote)
if revision.startswith('origin/'):
revision = 'refs/remotes/' + revision
if managed and platform.system() == 'Windows':
self._DisableHooks()
printed_path = False
verbose = []
if options.verbose:
self.Print('_____ %s at %s' % (self.relpath, revision),
timestamp=False)
verbose = ['--verbose']
printed_path = True
revision_ref = revision
if ':' in revision:
revision_ref, _, revision = revision.partition(':')
if revision_ref.startswith('refs/branch-heads'):
options.with_branch_heads = True
mirror = self._GetMirror(url, options, revision, revision_ref)
if mirror:
url = mirror.mirror_path
remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote)
if remote_ref:
# Rewrite remote refs to their local equivalents.
revision = ''.join(remote_ref)
rev_type = "branch"
elif revision.startswith('refs/'):
# Local branch? We probably don't want to support, since DEPS should
# always specify branches as they are in the upstream repo.
rev_type = "branch"
else:
# hash is also a tag, only make a distinction at checkout
rev_type = "hash"
# If we are going to introduce a new project, there is a possibility
# that we are syncing back to a state where the project was originally a
# sub-project rolled by DEPS (realistic case: crossing the Blink merge
# point syncing backwards, when Blink was a DEPS entry and not part of
# src.git). In such case, we might have a backup of the former .git
# folder, which can be used to avoid re-fetching the entire repo again
# (useful for bisects).
backup_dir = self.GetGitBackupDirPath()
target_dir = os.path.join(self.checkout_path, '.git')
if os.path.exists(backup_dir) and not os.path.exists(target_dir):
gclient_utils.safe_makedirs(self.checkout_path)
os.rename(backup_dir, target_dir)
# Reset to a clean state
self._Scrub('HEAD', options)
if (not os.path.exists(self.checkout_path) or
(os.path.isdir(self.checkout_path)
and not os.path.exists(os.path.join(self.checkout_path, '.git')))):
if mirror:
self._UpdateMirrorIfNotContains(mirror, options, rev_type,
revision)
try:
self._Clone(revision, url, options)
except subprocess2.CalledProcessError as e:
logging.warning('Clone failed due to: %s', e)
self._DeleteOrMove(options.force)
self._Clone(revision, url, options)
if file_list is not None:
files = self._Capture(
['-c', 'core.quotePath=false', 'ls-files']).splitlines()
file_list.extend(
[os.path.join(self.checkout_path, f) for f in files])
if mirror:
self._Capture(
['remote', 'set-url', '--push', 'origin', mirror.url])
if not verbose:
# Make the output a little prettier. It's nice to have some
# whitespace between projects when cloning.
self.Print('')
return self._Capture(['rev-parse', '--verify', 'HEAD'])
if mirror:
self._Capture(['remote', 'set-url', '--push', 'origin', mirror.url])
if not managed:
self._SetFetchConfig(options)
self.Print('________ unmanaged solution; skipping %s' %
self.relpath)
return self._Capture(['rev-parse', '--verify', 'HEAD'])
self._maybe_break_locks(options)
if mirror:
self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision)
# See if the url has changed (the unittests use git://foo for the url,
# let that through).
current_url = self._Capture(['config', 'remote.%s.url' % self.remote])
return_early = False
# TODO(maruel): Delete url != 'git://foo' since it's just to make the
# unit test pass. (and update the comment above)
# Skip url auto-correction if remote.origin.gclient-auto-fix-url is set.
# This allows devs to use experimental repos which have a different url
# but whose branch(s) are the same as official repos.
strp_url = url[:-4] if url.endswith('.git') else url
strp_current_url = current_url[:-4] if current_url.endswith(
'.git') else current_url
if (strp_current_url.rstrip('/') != strp_url.rstrip('/')
and url != 'git://foo' and
subprocess2.capture([
'git', 'config',
'remote.%s.gclient-auto-fix-url' % self.remote
],
cwd=self.checkout_path).strip() != 'False'):
self.Print('_____ switching %s from %s to new upstream %s' %
(self.relpath, current_url, url))
if not (options.force or options.reset):
# Make sure it's clean
self._CheckClean(revision)
# Switch over to the new upstream
self._Run(['remote', 'set-url', self.remote, url], options)
if mirror:
if git_cache.Mirror.CacheDirToUrl(current_url.rstrip(
'/')) == git_cache.Mirror.CacheDirToUrl(
url.rstrip('/')):
# Reset alternates when the cache dir is updated.
with open(
os.path.join(self.checkout_path, '.git', 'objects',
'info', 'alternates'), 'w') as fh:
fh.write(os.path.join(url, 'objects'))
else:
# Because we use Git alternatives, our existing repository
# is not self-contained. It's possible that new git
# alternative doesn't have all necessary objects that the
# current repository needs. Instead of blindly hoping that
# new alternative contains all necessary objects, keep the
# old alternative and just append a new one on top of it.
with open(
os.path.join(self.checkout_path, '.git', 'objects',
'info', 'alternates'), 'a') as fh:
fh.write("\n" + os.path.join(url, 'objects'))
self._EnsureValidHeadObjectOrCheckout(revision, options, url)
self._FetchAndReset(revision, file_list, options)
return_early = True
else:
self._EnsureValidHeadObjectOrCheckout(revision, options, url)
if return_early:
return self._Capture(['rev-parse', '--verify', 'HEAD'])
cur_branch = self._GetCurrentBranch()
# Cases:
# 0) HEAD is detached. Probably from our initial clone.
# - make sure HEAD is contained by a named ref, then update.
# Cases 1-4. HEAD is a branch.
# 1) current branch is not tracking a remote branch
# - try to rebase onto the new hash or branch
# 2) current branch is tracking a remote branch with local committed
# changes, but the DEPS file switched to point to a hash
# - rebase those changes on top of the hash
# 3) current branch is tracking a remote branch w/or w/out changes, and
# no DEPS switch
# - see if we can FF, if not, prompt the user for rebase, merge, or stop
# 4) current branch is tracking a remote branch, but DEPS switches to a
# different remote branch, and a) current branch has no local changes,
# and --force: - checkout new branch b) current branch has local
# changes, and --force and --reset: - checkout new branch c) otherwise
# exit
# GetUpstreamBranch returns something like 'refs/remotes/origin/main'
# for a tracking branch or 'main' if not a tracking branch (it's based
# on a specific rev/hash) or it returns None if it couldn't find an
# upstream
if cur_branch is None:
upstream_branch = None
current_type = "detached"
logging.debug("Detached HEAD")
else:
upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path)
if not upstream_branch or not upstream_branch.startswith(
'refs/remotes'):
current_type = "hash"
logging.debug(
"Current branch is not tracking an upstream (remote)"
" branch.")
elif upstream_branch.startswith('refs/remotes'):
current_type = "branch"
else:
raise gclient_utils.Error('Invalid Upstream: %s' %
upstream_branch)
self._SetFetchConfig(options)
# Fetch upstream if we don't already have |revision|.
if not scm.GIT.IsValidRevision(
self.checkout_path, revision, sha_only=True):
self._Fetch(options, prune=options.force)
if not scm.GIT.IsValidRevision(
self.checkout_path, revision, sha_only=True):
# Update the remotes first so we have all the refs.
remote_output = scm.GIT.Capture(['remote'] + verbose +
['update'],
cwd=self.checkout_path)
if verbose:
self.Print(remote_output)
revision = self._AutoFetchRef(options, revision)
# This is a big hammer, debatable if it should even be here...
if options.force or options.reset:
target = 'HEAD'
if options.upstream and upstream_branch:
target = upstream_branch
self._Scrub(target, options)
if current_type == 'detached':
# case 0
# We just did a Scrub, this is as clean as it's going to get. In
# particular if HEAD is a commit that contains two versions of the
# same file on a case-insensitive filesystem (e.g. 'a' and 'A'),
# there's no way to actually "Clean" the checkout; that commit is
# uncheckoutable on this system. The best we can do is carry forward
# to the checkout step.
if not (options.force or options.reset):
self._CheckClean(revision)
self._CheckDetachedHead(revision, options)
if self._Capture(['rev-list', '-n', '1', 'HEAD']) == revision:
self.Print('Up-to-date; skipping checkout.')
else:
# 'git checkout' may need to overwrite existing untracked files.
# Allow it only when nuclear options are enabled.
self._Checkout(
options,
revision,
force=(options.force and options.delete_unversioned_trees),
quiet=True,
)
if not printed_path:
self.Print('_____ %s at %s' % (self.relpath, revision),
timestamp=False)
elif current_type == 'hash':
# case 1
# Can't find a merge-base since we don't know our upstream. That
# makes this command VERY likely to produce a rebase failure. For
# now we assume origin is our upstream since that's what the old
# behavior was.
upstream_branch = self.remote
if options.revision or deps_revision:
upstream_branch = revision
self._AttemptRebase(upstream_branch,
file_list,
options,
printed_path=printed_path,
merge=options.merge)
printed_path = True
elif rev_type == 'hash':
# case 2
self._AttemptRebase(upstream_branch,
file_list,
options,
newbase=revision,
printed_path=printed_path,
merge=options.merge)
printed_path = True
elif remote_ref and ''.join(remote_ref) != upstream_branch:
# case 4
new_base = ''.join(remote_ref)
if not printed_path:
self.Print('_____ %s at %s' % (self.relpath, revision),
timestamp=False)
switch_error = (
"Could not switch upstream branch from %s to %s\n" %
(upstream_branch, new_base) +
"Please use --force or merge or rebase manually:\n" +
"cd %s; git rebase %s\n" % (self.checkout_path, new_base) +
"OR git checkout -b <some new branch> %s" % new_base)
force_switch = False
if options.force:
try:
self._CheckClean(revision)
# case 4a
force_switch = True
except gclient_utils.Error as e:
if options.reset:
# case 4b
force_switch = True
else:
switch_error = '%s\n%s' % (str(e), switch_error)
if force_switch:
self.Print("Switching upstream branch from %s to %s" %
(upstream_branch, new_base))
switch_branch = 'gclient_' + remote_ref[1]
self._Capture(['branch', '-f', switch_branch, new_base])
self._Checkout(options, switch_branch, force=True, quiet=True)
else:
# case 4c
raise gclient_utils.Error(switch_error)
else:
# case 3 - the default case
rebase_files = self._GetDiffFilenames(upstream_branch)
if verbose:
self.Print('Trying fast-forward merge to branch : %s' %
upstream_branch)
try:
merge_args = ['merge']
if options.merge:
merge_args.append('--ff')
else:
merge_args.append('--ff-only')
merge_args.append(upstream_branch)
merge_output = self._Capture(merge_args)
except subprocess2.CalledProcessError as e:
rebase_files = []
if re.search(b'fatal: Not possible to fast-forward, aborting.',
e.stderr):
if not printed_path:
self.Print('_____ %s at %s' % (self.relpath, revision),
timestamp=False)
printed_path = True
while True:
if not options.auto_rebase:
try:
action = self._AskForData(
'Cannot %s, attempt to rebase? '
'(y)es / (q)uit / (s)kip : ' %
('merge' if options.merge else
'fast-forward merge'), options)
except ValueError:
raise gclient_utils.Error('Invalid Character')
if options.auto_rebase or re.match(
r'yes|y', action, re.I):
self._AttemptRebase(upstream_branch,
rebase_files,
options,
printed_path=printed_path,
merge=False)
printed_path = True
break
if re.match(r'quit|q', action, re.I):
raise gclient_utils.Error(
"Can't fast-forward, please merge or "
"rebase manually.\n"
"cd %s && git " % self.checkout_path +
"rebase %s" % upstream_branch)
if re.match(r'skip|s', action, re.I):
self.Print('Skipping %s' % self.relpath)
return
self.Print('Input not recognized')
elif re.match(
b"error: Your local changes to '.*' would be "
b"overwritten by merge. Aborting.\nPlease, commit your "
b"changes or stash them before you can merge.\n",
e.stderr):
if not printed_path:
self.Print('_____ %s at %s' % (self.relpath, revision),
timestamp=False)
printed_path = True
raise gclient_utils.Error(e.stderr.decode('utf-8'))
else:
# Some other problem happened with the merge
logging.error("Error during fast-forward merge in %s!" %
self.relpath)
self.Print(e.stderr.decode('utf-8'))
raise
else:
# Fast-forward merge was successful
if not re.match('Already up-to-date.', merge_output) or verbose:
if not printed_path:
self.Print('_____ %s at %s' % (self.relpath, revision),
timestamp=False)
printed_path = True
self.Print(merge_output.strip())
if not verbose:
# Make the output a little prettier. It's nice to have
# some whitespace between projects when syncing.
self.Print('')
if file_list is not None:
file_list.extend(
[os.path.join(self.checkout_path, f) for f in rebase_files])
# If the rebase generated a conflict, abort and ask user to fix
if self._IsRebasing():
raise gclient_utils.Error(
'\n____ %s at %s\n'
'\nConflict while rebasing this branch.\n'
'Fix the conflict and run gclient again.\n'
'See man git-rebase for details.\n' % (self.relpath, revision))
if verbose:
self.Print('Checked out revision %s' %
self.revinfo(options, (), None),
timestamp=False)
# If --reset and --delete_unversioned_trees are specified, remove any
# untracked directories.
if options.reset and options.delete_unversioned_trees:
# GIT.CaptureStatus() uses 'dit diff' to compare to a specific SHA1
# (the merge-base by default), so doesn't include untracked files.
# So we use 'git ls-files --directory --others --exclude-standard'
# here directly.
paths = scm.GIT.Capture([
'-c', 'core.quotePath=false', 'ls-files', '--directory',
'--others', '--exclude-standard'
], self.checkout_path)
for path in (p for p in paths.splitlines() if p.endswith('/')):
full_path = os.path.join(self.checkout_path, path)
if not os.path.islink(full_path):
self.Print('_____ removing unversioned directory %s' % path)
gclient_utils.rmtree(full_path)
return self._Capture(['rev-parse', '--verify', 'HEAD'])
def revert(self, options, _args, file_list):
"""Reverts local modifications.
All reverted files will be appended to file_list.
"""
if not os.path.isdir(self.checkout_path):
# revert won't work if the directory doesn't exist. It needs to
# checkout instead.
self.Print('_____ %s is missing, syncing instead' % self.relpath)
# Don't reuse the args.
return self.update(options, [], file_list)
default_rev = "refs/heads/main"
if options.upstream:
if self._GetCurrentBranch():
upstream_branch = scm.GIT.GetUpstreamBranch(self.checkout_path)
default_rev = upstream_branch or default_rev
_, deps_revision = gclient_utils.SplitUrlRevision(self.url)
if not deps_revision:
deps_revision = default_rev
if deps_revision.startswith('refs/heads/'):
deps_revision = deps_revision.replace('refs/heads/',
self.remote + '/')
try:
deps_revision = self.GetUsableRev(deps_revision, options)
except NoUsableRevError as e:
# If the DEPS entry's url and hash changed, try to update the
# origin. See also http://crbug.com/520067.
logging.warning(
"Couldn't find usable revision, will retrying to update instead: %s",
str(e))
return self.update(options, [], file_list)
if file_list is not None:
files = self._GetDiffFilenames(deps_revision)
self._Scrub(deps_revision, options)
self._Run(['clean', '-f', '-d'], options)
if file_list is not None:
file_list.extend(
[os.path.join(self.checkout_path, f) for f in files])
def revinfo(self, _options, _args, _file_list):
"""Returns revision"""
return self._Capture(['rev-parse', 'HEAD'])
def runhooks(self, options, args, file_list):
self.status(options, args, file_list)
def status(self, options, _args, file_list):
"""Display status information."""
if not os.path.isdir(self.checkout_path):
self.Print('________ couldn\'t run status in %s:\n'
'The directory does not exist.' % self.checkout_path)
else:
merge_base = []
if self.url:
_, base_rev = gclient_utils.SplitUrlRevision(self.url)
if base_rev:
if base_rev.startswith('refs/'):
base_rev = self._ref_to_remote_ref(base_rev)
merge_base = [base_rev]
self._Run(['-c', 'core.quotePath=false', 'diff', '--name-status'] +
merge_base,
options,
always_show_header=options.verbose)
if file_list is not None:
files = self._GetDiffFilenames(
merge_base[0] if merge_base else None)
file_list.extend(
[os.path.join(self.checkout_path, f) for f in files])
def GetUsableRev(self, rev, options):
"""Finds a useful revision for this repository."""
sha1 = None
if not os.path.isdir(self.checkout_path):
raise NoUsableRevError(
'This is not a git repo, so we cannot get a usable rev.')
if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev):
sha1 = rev
else:
# May exist in origin, but we don't have it yet, so fetch and look
# again.
self._Fetch(options)
if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev):
sha1 = rev
if not sha1:
raise NoUsableRevError(
'Hash %s does not appear to be a valid hash in this repo.' %
rev)
return sha1
def GetGitBackupDirPath(self):
"""Returns the path where the .git folder for the current project can be
staged/restored. Use case: subproject moved from DEPS <-> outer project."""
return os.path.join(self._root_dir,
'old_' + self.relpath.replace(os.sep, '_')) + '.git'
def _GetMirror(self, url, options, revision=None, revision_ref=None):
"""Get a git_cache.Mirror object for the argument url."""
if not self.cache_dir:
return None
mirror_kwargs = {
'print_func': self.filter,
'refs': [],
'commits': [],
}
if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
mirror_kwargs['refs'].append('refs/branch-heads/*')
elif revision_ref and revision_ref.startswith('refs/branch-heads/'):
mirror_kwargs['refs'].append(revision_ref)
if hasattr(options, 'with_tags') and options.with_tags:
mirror_kwargs['refs'].append('refs/tags/*')
elif revision_ref and revision_ref.startswith('refs/tags/'):
mirror_kwargs['refs'].append(revision_ref)
if revision and not revision.startswith('refs/'):
mirror_kwargs['commits'].append(revision)
return git_cache.Mirror(url, **mirror_kwargs)
def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision):
"""Update a git mirror by fetching the latest commits from the remote,
unless mirror already contains revision whose type is sha1 hash.
"""
if rev_type == 'hash' and mirror.contains_revision(revision):
if options.verbose:
self.Print('skipping mirror update, it has rev=%s already' %
revision,
timestamp=False)
return
if getattr(options, 'shallow', False):
depth = 10000
else:
depth = None
mirror.populate(verbose=options.verbose,
bootstrap=not getattr(options, 'no_bootstrap', False),
depth=depth,
lock_timeout=getattr(options, 'lock_timeout', 0))
def _Clone(self, revision, url, options):
"""Clone a git repository from the given URL.
Once we've cloned the repo, we checkout a working branch if the specified
revision is a branch head. If it is a tag or a specific commit, then we
leave HEAD detached as it makes future updates simpler -- in this case the
user should first create a new branch or switch to an existing branch before
making changes in the repo."""
in_cog_workspace = self._IsCog()
if self.print_outbuf:
print_stdout = True
filter_fn = None
else:
print_stdout = False
filter_fn = self.filter
if not options.verbose:
# git clone doesn't seem to insert a newline properly before
# printing to stdout
self.Print('')
# If the parent directory does not exist, Git clone on Windows will not
# create it, so we need to do it manually.
parent_dir = os.path.dirname(self.checkout_path)
gclient_utils.safe_makedirs(parent_dir)
if in_cog_workspace:
clone_cmd = ['citc', 'clone-repo', url, self.checkout_path]
clone_cmd.append(
gclient_utils.ExtractRefName(self.remote, revision) or revision)
try:
self._Run(clone_cmd,
options,
cwd=self._root_dir,
retry=True,
print_stdout=print_stdout,
filter_fn=filter_fn)
except:
traceback.print_exc(file=self.out_fh)
raise
self._SetFetchConfig(options)
elif hasattr(options, 'no_history') and options.no_history:
self._Run(['init', self.checkout_path], options, cwd=self._root_dir)
self._Run(['remote', 'add', 'origin', url], options)
revision = self._AutoFetchRef(options, revision, depth=1)
remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote)
self._Checkout(options, ''.join(remote_ref or revision), quiet=True)
else:
cfg = gclient_utils.DefaultIndexPackConfig(url)
clone_cmd = cfg + ['clone', '--no-checkout', '--progress']
if self.cache_dir:
clone_cmd.append('--shared')
if options.verbose:
clone_cmd.append('--verbose')
clone_cmd.append(url)
tmp_dir = tempfile.mkdtemp(prefix='_gclient_%s_' %
os.path.basename(self.checkout_path),
dir=parent_dir)
clone_cmd.append(tmp_dir)
try:
self._Run(clone_cmd,
options,
cwd=self._root_dir,
retry=True,
print_stdout=print_stdout,
filter_fn=filter_fn)
logging.debug(
'Cloned into temporary dir, moving to checkout_path')
gclient_utils.safe_makedirs(self.checkout_path)
gclient_utils.safe_rename(
os.path.join(tmp_dir, '.git'),
os.path.join(self.checkout_path, '.git'))
except:
traceback.print_exc(file=self.out_fh)
raise
finally:
if os.listdir(tmp_dir):
self.Print('_____ removing non-empty tmp dir %s' % tmp_dir)
gclient_utils.rmtree(tmp_dir)
self._SetFetchConfig(options)
self._Fetch(options, prune=options.force)
revision = self._AutoFetchRef(options, revision)
remote_ref = scm.GIT.RefToRemoteRef(revision, self.remote)
self._Checkout(options, ''.join(remote_ref or revision), quiet=True)
if self._GetCurrentBranch() is None:
# Squelch git's very verbose detached HEAD warning and use our own
self.Print((
'Checked out %s to a detached HEAD. Before making any commits\n'
'in this repo, you should use \'git checkout <branch>\' to switch \n'
'to an existing branch or use \'git checkout %s -b <branch>\' to\n'
'create a new branch for your work.') % (revision, self.remote))
def _AskForData(self, prompt, options):
if options.jobs > 1:
self.Print(prompt)
raise gclient_utils.Error("Background task requires input. Rerun "
"gclient with --jobs=1 so that\n"
"interaction is possible.")
return gclient_utils.AskForData(prompt)
def _AttemptRebase(self,
upstream,
files,
options,
newbase=None,
branch=None,
printed_path=False,
merge=False):
"""Attempt to rebase onto either upstream or, if specified, newbase."""
if files is not None:
files.extend(self._GetDiffFilenames(upstream))
revision = upstream
if newbase:
revision = newbase
action = 'merge' if merge else 'rebase'
if not printed_path:
self.Print('_____ %s : Attempting %s onto %s...' %
(self.relpath, action, revision))
printed_path = True
else:
self.Print('Attempting %s onto %s...' % (action, revision))
if merge:
merge_output = self._Capture(['merge', revision])
if options.verbose:
self.Print(merge_output)
return
# Build the rebase command here using the args
# git rebase [options] [--onto <newbase>] <upstream> [<branch>]
rebase_cmd = ['rebase']
if options.verbose:
rebase_cmd.append('--verbose')
if newbase:
rebase_cmd.extend(['--onto', newbase])
rebase_cmd.append(upstream)
if branch:
rebase_cmd.append(branch)
try:
rebase_output = scm.GIT.Capture(rebase_cmd, cwd=self.checkout_path)
except subprocess2.CalledProcessError as e:
if (re.match(
br'cannot rebase: you have unstaged changes', e.stderr
) or re.match(
br'cannot rebase: your index contains uncommitted changes',
e.stderr)):
while True:
rebase_action = self._AskForData(
'Cannot rebase because of unstaged changes.\n'
'\'git reset --hard HEAD\' ?\n'
'WARNING: destroys any uncommitted work in your current branch!'
' (y)es / (q)uit / (s)how : ', options)
if re.match(r'yes|y', rebase_action, re.I):
self._Scrub('HEAD', options)
# Should this be recursive?
rebase_output = scm.GIT.Capture(rebase_cmd,
cwd=self.checkout_path)
break
if re.match(r'quit|q', rebase_action, re.I):
raise gclient_utils.Error(
"Please merge or rebase manually\n"
"cd %s && git " % self.checkout_path +
"%s" % ' '.join(rebase_cmd))
if re.match(r'show|s', rebase_action, re.I):
self.Print('%s' % e.stderr.decode('utf-8').strip())
continue
gclient_utils.Error("Input not recognized")
continue
elif re.search(br'^CONFLICT', e.stdout, re.M):
raise gclient_utils.Error(
"Conflict while rebasing this branch.\n"
"Fix the conflict and run gclient again.\n"
"See 'man git-rebase' for details.\n")
else:
self.Print(e.stdout.decode('utf-8').strip())
self.Print('Rebase produced error output:\n%s' %
e.stderr.decode('utf-8').strip())
raise gclient_utils.Error(
"Unrecognized error, please merge or rebase "
"manually.\ncd %s && git " % self.checkout_path +
"%s" % ' '.join(rebase_cmd))
self.Print(rebase_output.strip())
if not options.verbose:
# Make the output a little prettier. It's nice to have some
# whitespace between projects when syncing.
self.Print('')
@staticmethod
def _CheckMinVersion(min_version):
(ok, current_version) = scm.GIT.AssertVersion(min_version)
if not ok:
raise gclient_utils.Error('git version %s < minimum required %s' %
(current_version, min_version))
def _EnsureValidHeadObjectOrCheckout(self, revision, options, url):
# Special case handling if all 3 conditions are met:
# * the mirros have recently changed, but deps destination remains same,
# * the git histories of mirrors are conflicting. * git cache is used
# This manifests itself in current checkout having invalid HEAD commit
# on most git operations. Since git cache is used, just deleted the .git
# folder, and re-create it by cloning.
try:
self._Capture(['rev-list', '-n', '1', 'HEAD'])
except subprocess2.CalledProcessError as e:
if (b'fatal: bad object HEAD' in e.stderr and self.cache_dir
and self.cache_dir in url):
self.Print(
('Likely due to DEPS change with git cache_dir, '
'the current commit points to no longer existing object.\n'
'%s' % e))
self._DeleteOrMove(options.force)
self._Clone(revision, url, options)
else:
raise
def _IsRebasing(self):
# Check for any of REBASE-i/REBASE-m/REBASE/AM. Unfortunately git
# doesn't have a plumbing command to determine whether a rebase is in
# progress, so for now emualate (more-or-less) git-rebase.sh /
# git-completion.bash
g = os.path.join(self.checkout_path, '.git')
return (os.path.isdir(os.path.join(g, "rebase-merge"))
or os.path.isdir(os.path.join(g, "rebase-apply")))
def _CheckClean(self, revision):
lockfile = os.path.join(self.checkout_path, ".git", "index.lock")
if os.path.exists(lockfile):
raise gclient_utils.Error(
'\n____ %s at %s\n'
'\tYour repo is locked, possibly due to a concurrent git process.\n'
'\tIf no git executable is running, then clean up %r and try again.\n'
% (self.relpath, revision, lockfile))
# Make sure the tree is clean; see git-rebase.sh for reference
try:
scm.GIT.Capture(
['update-index', '--ignore-submodules', '--refresh'],
cwd=self.checkout_path)
except subprocess2.CalledProcessError:
raise gclient_utils.Error(
'\n____ %s at %s\n'
'\tYou have unstaged changes.\n'
'\tcd into %s, run git status to see changes,\n'
'\tand commit, stash, or reset.\n' %
(self.relpath, revision, self.relpath))
try:
scm.GIT.Capture([
'diff-index', '--cached', '--name-status', '-r',
'--ignore-submodules', 'HEAD', '--'
],
cwd=self.checkout_path)
except subprocess2.CalledProcessError:
raise gclient_utils.Error(
'\n____ %s at %s\n'
'\tYour index contains uncommitted changes\n'
'\tcd into %s, run git status to see changes,\n'
'\tand commit, stash, or reset.\n' %
(self.relpath, revision, self.relpath))
def _CheckDetachedHead(self, revision, _options):
# HEAD is detached. Make sure it is safe to move away from (i.e., it is
# reference by a commit). If not, error out -- most likely a rebase is
# in progress, try to detect so we can give a better error.
try:
scm.GIT.Capture(['name-rev', '--no-undefined', 'HEAD'],
cwd=self.checkout_path)
except subprocess2.CalledProcessError:
# Commit is not contained by any rev. See if the user is rebasing:
if self._IsRebasing():
# Punt to the user
raise gclient_utils.Error(
'\n____ %s at %s\n'
'\tAlready in a conflict, i.e. (no branch).\n'
'\tFix the conflict and run gclient again.\n'
'\tOr to abort run:\n\t\tgit-rebase --abort\n'
'\tSee man git-rebase for details.\n' %
(self.relpath, revision))
# Let's just save off the commit so we can proceed.
name = ('saved-by-gclient-' +
self._Capture(['rev-parse', '--short', 'HEAD']))
self._Capture(['branch', '-f', name])
self.Print(
'_____ found an unreferenced commit and saved it as \'%s\'' %
name)
def _GetCurrentBranch(self):
# Returns name of current branch or None for detached HEAD
branch = self._Capture(['rev-parse', '--abbrev-ref=strict', 'HEAD'])
if branch == 'HEAD':
return None
return branch
def _Capture(self, args, **kwargs):
set_git_dir = 'cwd' not in kwargs
kwargs.setdefault('cwd', self.checkout_path)
kwargs.setdefault('stderr', subprocess2.PIPE)
strip = kwargs.pop('strip', True)
env = scm.GIT.ApplyEnvVars(kwargs)
# If an explicit cwd isn't set, then default to the .git/ subdir so we
# get stricter behavior. This can be useful in cases of slight
# corruption -- we don't accidentally go corrupting parent git checks
# too. See https://crbug.com/1000825 for an example.
if set_git_dir:
env.setdefault(
'GIT_DIR',
os.path.abspath(os.path.join(self.checkout_path, '.git')))
ret = subprocess2.check_output(['git'] + args, env=env,
**kwargs).decode('utf-8')
if strip:
ret = ret.strip()
self.Print('Finished running: %s %s' % ('git', ' '.join(args)))
return ret
def _Checkout(self, options, ref, force=False, quiet=None):
"""Performs a 'git-checkout' operation.
Args:
options: The configured option set
ref: (str) The branch/commit to checkout
quiet: (bool/None) Whether or not the checkout should pass '--quiet'; if
'None', the behavior is inferred from 'options.verbose'.
Returns: (str) The output of the checkout operation
"""
if quiet is None:
quiet = (not options.verbose)
checkout_args = ['checkout']
if force:
checkout_args.append('--force')
if quiet:
checkout_args.append('--quiet')
checkout_args.append(ref)
return self._Capture(checkout_args)
def _Fetch(self,
options,
remote=None,
prune=False,
quiet=False,
refspec=None,
depth=None):
cfg = gclient_utils.DefaultIndexPackConfig(self.url)
# When updating, the ref is modified to be a remote ref .
# (e.g. refs/heads/NAME becomes refs/remotes/REMOTE/NAME).
# Try to reverse that mapping.
original_ref = scm.GIT.RemoteRefToRef(refspec, self.remote)
if original_ref:
refspec = original_ref + ':' + refspec
# When a mirror is configured, it only fetches
# refs/{heads,branch-heads,tags}/*.
# If asked to fetch other refs, we must fetch those directly from
# the repository, and not from the mirror.
if not original_ref.startswith(
('refs/heads/', 'refs/branch-heads/', 'refs/tags/')):
remote, _ = gclient_utils.SplitUrlRevision(self.url)
fetch_cmd = cfg + [
'fetch',
remote or self.remote,
]
if refspec:
fetch_cmd.append(refspec)
if prune:
fetch_cmd.append('--prune')
if options.verbose:
fetch_cmd.append('--verbose')
if not hasattr(options, 'with_tags') or not options.with_tags:
fetch_cmd.append('--no-tags')
elif quiet:
fetch_cmd.append('--quiet')
if depth:
fetch_cmd.append('--depth=' + str(depth))
self._Run(fetch_cmd, options, show_header=options.verbose, retry=True)
def _SetFetchConfig(self, options):
"""Adds, and optionally fetches, "branch-heads" and "tags" refspecs
if requested."""
if options.force or options.reset:
try:
self._Run(
['config', '--unset-all',
'remote.%s.fetch' % self.remote], options)
self._Run([
'config',
'remote.%s.fetch' % self.remote,
'+refs/heads/*:refs/remotes/%s/*' % self.remote
], options)
except subprocess2.CalledProcessError as e:
# If exit code was 5, it means we attempted to unset a config
# that didn't exist. Ignore it.
if e.returncode != 5:
raise
if hasattr(options, 'with_branch_heads') and options.with_branch_heads:
config_cmd = [
'config',
'remote.%s.fetch' % self.remote,
'+refs/branch-heads/*:refs/remotes/branch-heads/*',
'^\\+refs/branch-heads/\\*:.*$'
]
self._Run(config_cmd, options)
if hasattr(options, 'with_tags') and options.with_tags:
config_cmd = [
'config',
'remote.%s.fetch' % self.remote, '+refs/tags/*:refs/tags/*',
'^\\+refs/tags/\\*:.*$'
]
self._Run(config_cmd, options)
def _AutoFetchRef(self, options, revision, depth=None):
"""Attempts to fetch |revision| if not available in local repo.
Returns possibly updated revision."""
if not scm.GIT.IsValidRevision(self.checkout_path, revision):
self._Fetch(options, refspec=revision, depth=depth)
revision = self._Capture(['rev-parse', 'FETCH_HEAD'])
return revision
def _Run(self, args, options, **kwargs):
# Disable 'unused options' warning | pylint: disable=unused-argument
kwargs.setdefault('cwd', self.checkout_path)
kwargs.setdefault('filter_fn', self.filter)
kwargs.setdefault('show_header', True)
env = scm.GIT.ApplyEnvVars(kwargs)
cmd = ['git'] + args
gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs)
class CipdPackage(object):
"""A representation of a single CIPD package."""
def __init__(self, name, version, authority_for_subdir):
self._authority_for_subdir = authority_for_subdir
self._name = name
self._version = version
@property
def authority_for_subdir(self):
"""Whether this package has authority to act on behalf of its subdir.
Some operations should only be performed once per subdirectory. A package
that has authority for its subdirectory is the only package that should
perform such operations.
Returns:
bool; whether this package has subdir authority.
"""
return self._authority_for_subdir
@property
def name(self):
return self._name
@property
def version(self):
return self._version
class CipdRoot(object):
"""A representation of a single CIPD root."""
def __init__(self, root_dir, service_url, log_level=None):
self._all_packages = set()
self._mutator_lock = threading.Lock()
self._packages_by_subdir = collections.defaultdict(list)
self._root_dir = root_dir
self._service_url = service_url
self._resolved_packages = None
self._log_level = log_level or 'error'
def add_package(self, subdir, package, version):
"""Adds a package to this CIPD root.
As far as clients are concerned, this grants both root and subdir authority
to packages arbitrarily. (The implementation grants root authority to the
first package added and subdir authority to the first package added for that
subdir, but clients should not depend on or expect that behavior.)
Args:
subdir: str; relative path to where the package should be installed from
the cipd root directory.
package: str; the cipd package name.
version: str; the cipd package version.
Returns:
CipdPackage; the package that was created and added to this root.
"""
with self._mutator_lock:
cipd_package = CipdPackage(package, version,
not self._packages_by_subdir[subdir])
self._all_packages.add(cipd_package)
self._packages_by_subdir[subdir].append(cipd_package)
return cipd_package
def packages(self, subdir):
"""Get the list of configured packages for the given subdir."""
return list(self._packages_by_subdir[subdir])
def resolved_packages(self):
if not self._resolved_packages:
self._resolved_packages = self.ensure_file_resolve()
return self._resolved_packages
def clobber(self):
"""Remove the .cipd directory.
This is useful for forcing ensure to redownload and reinitialize all
packages.
"""
with self._mutator_lock:
cipd_cache_dir = os.path.join(self.root_dir, '.cipd')
try:
gclient_utils.rmtree(os.path.join(cipd_cache_dir))
except OSError:
if os.path.exists(cipd_cache_dir):
raise
def expand_package_name(self, package_name_string, **kwargs):
"""Run `cipd expand-package-name`.
CIPD package names can be declared with placeholder variables
such as '${platform}', this cmd will return the package name
with the variables resolved. The resolution is based on the host
the command is executing on.
"""
kwargs.setdefault('stderr', subprocess2.PIPE)
cmd = ['cipd', 'expand-package-name', package_name_string]
ret = subprocess2.check_output(cmd, **kwargs).decode('utf-8')
return ret.strip()
@contextlib.contextmanager
def _create_ensure_file(self):
try:
contents = '$ParanoidMode CheckPresence\n'
# TODO(crbug/1329641): Remove once cipd packages have been updated
# to always be created in copy mode.
contents += '$OverrideInstallMode copy\n\n'
for subdir, packages in sorted(self._packages_by_subdir.items()):
contents += '@Subdir %s\n' % subdir
for package in sorted(packages, key=lambda p: p.name):
contents += '%s %s\n' % (package.name, package.version)
contents += '\n'
ensure_file = None
with tempfile.NamedTemporaryFile(suffix='.ensure',
delete=False,
mode='wb') as ensure_file:
ensure_file.write(contents.encode('utf-8', 'replace'))
yield ensure_file.name
finally:
if ensure_file is not None and os.path.exists(ensure_file.name):
os.remove(ensure_file.name)
def ensure(self):
"""Run `cipd ensure`."""
with self._mutator_lock:
with self._create_ensure_file() as ensure_file:
cmd = [
'cipd',
'ensure',
'-log-level',
self._log_level,
'-root',
self.root_dir,
'-ensure-file',
ensure_file,
]
gclient_utils.CheckCallAndFilter(cmd,
print_stdout=True,
show_header=True)
@contextlib.contextmanager
def _create_ensure_file_for_resolve(self):
try:
contents = '$ResolvedVersions %s\n' % os.devnull
for subdir, packages in sorted(self._packages_by_subdir.items()):
contents += '@Subdir %s\n' % subdir
for package in sorted(packages, key=lambda p: p.name):
contents += '%s %s\n' % (package.name, package.version)
contents += '\n'
ensure_file = None
with tempfile.NamedTemporaryFile(suffix='.ensure',
delete=False,
mode='wb') as ensure_file:
ensure_file.write(contents.encode('utf-8', 'replace'))
yield ensure_file.name
finally:
if ensure_file is not None and os.path.exists(ensure_file.name):
os.remove(ensure_file.name)
def _create_resolved_file(self):
return tempfile.NamedTemporaryFile(suffix='.resolved',
delete=False,
mode='wb')
def ensure_file_resolve(self):
"""Run `cipd ensure-file-resolve`."""
with self._mutator_lock:
with self._create_resolved_file() as output_file:
with self._create_ensure_file_for_resolve() as ensure_file:
cmd = [
'cipd',
'ensure-file-resolve',
'-log-level',
self._log_level,
'-ensure-file',
ensure_file,
'-json-output',
output_file.name,
]
gclient_utils.CheckCallAndFilter(cmd,
print_stdout=False,
show_header=False)
with open(output_file.name) as f:
output_json = json.load(f)
return output_json.get('result', {})
def run(self, command):
if command == 'update':
self.ensure()
elif command == 'revert':
self.clobber()
self.ensure()
def created_package(self, package):
"""Checks whether this root created the given package.
Args:
package: CipdPackage; the package to check.
Returns:
bool; whether this root created the given package.
"""
return package in self._all_packages
@property
def root_dir(self):
return self._root_dir
@property
def service_url(self):
return self._service_url
class CipdWrapper(SCMWrapper):
"""Wrapper for CIPD.
Currently only supports chrome-infra-packages.appspot.com.
"""
name = 'cipd'
def __init__(self,
url=None,
root_dir=None,
relpath=None,
out_fh=None,
out_cb=None,
root=None,
package=None):
super(CipdWrapper, self).__init__(url=url,
root_dir=root_dir,
relpath=relpath,
out_fh=out_fh,
out_cb=out_cb)
assert root.created_package(package)
self._package = package
self._root = root
#override
def GetCacheMirror(self):
return None
#override
def GetActualRemoteURL(self, options):
return self._root.service_url
#override
def DoesRemoteURLMatch(self, options):
del options
return True
def revert(self, options, args, file_list):
"""Does nothing.
CIPD packages should be reverted at the root by running
`CipdRoot.run('revert')`.
"""
def diff(self, options, args, file_list):
"""CIPD has no notion of diffing."""
def pack(self, options, args, file_list):
"""CIPD has no notion of diffing."""
def revinfo(self, options, args, file_list):
"""Grab the instance ID."""
try:
tmpdir = tempfile.mkdtemp()
# Attempt to get instance_id from the root resolved cache.
# Resolved cache will not match on any CIPD packages with
# variables such as ${platform}, they will fall back to
# the slower method below.
resolved = self._root.resolved_packages()
if resolved:
# CIPD uses POSIX separators across all platforms, so
# replace any Windows separators.
path_split = self.relpath.replace(os.sep, "/").split(":")
if len(path_split) > 1:
src_path, package = path_split
if src_path in resolved:
for resolved_package in resolved[src_path]:
if package == resolved_package.get(
'pin', {}).get('package'):
return resolved_package.get(
'pin', {}).get('instance_id')
describe_json_path = os.path.join(tmpdir, 'describe.json')
cmd = [
'cipd', 'describe', self._package.name, '-log-level', 'error',
'-version', self._package.version, '-json-output',
describe_json_path
]
gclient_utils.CheckCallAndFilter(cmd)
with open(describe_json_path) as f:
describe_json = json.load(f)
return describe_json.get('result', {}).get('pin',
{}).get('instance_id')
finally:
gclient_utils.rmtree(tmpdir)
def status(self, options, args, file_list):
pass
def update(self, options, args, file_list):
"""Does nothing.
CIPD packages should be updated at the root by running
`CipdRoot.run('update')`.
"""
此处可能存在不合适展示的内容，页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容，可点击提交进行申诉，我们将尽快为您处理。
马建仓 AI 助手
