gitscraper: Provide way to get timestamps for files.

When scraping files from the repository allow passing an additional list of
files for which the timestamp of the last change is retrieved. Since this is
slow (the log needs to be traversed, since a blob doesn't know about commits
its referenced by) only specified files get this retrieved.

Change-Id: I61a2ef812ac18ffb5898c01ce070046d4ca00a15
This commit is contained in:
Dominik Riebeling 2012-04-29 11:38:23 +02:00
parent a8b597e620
commit 91cf4a7768

View file

@ -42,8 +42,8 @@ def get_refs(repo):
@return Dict matching hashes to each ref.
'''
print("Getting list of refs")
output = subprocess.Popen(["git", "show-ref"], stdout=subprocess.PIPE,
stderr=subprocess.PIPE, cwd=repo)
output = subprocess.Popen(["git", "show-ref", "--abbrev"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
cmdout = output.communicate()
refs = {}
@ -99,6 +99,21 @@ def get_lstree(repo, start, filterlist=[]):
return objects
def get_file_timestamp(repo, tree, filename):
'''Get timestamp for a file.
@param repo Path to repository root.
@param tree Hash of tree to use.
@param filename Filename in tree
@return Timestamp as string.
'''
output = subprocess.Popen(
["git", "log", "--format=%ai", "-n", "1", tree, filename],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repo)
cmdout = output.communicate()
return cmdout[0].decode().rstrip()
def get_object(repo, blob, destfile):
'''Get an identified object from the repository.
@param repo Path to repository root.
@ -138,7 +153,7 @@ def describe_treehash(repo, treehash):
return cmdout[0].rstrip()
def scrape_files(repo, treehash, filelist, dest=""):
def scrape_files(repo, treehash, filelist, dest="", timestamp_files=[]):
'''Scrape list of files from repository.
@param repo Path to repository root.
@param treehash Hash identifying the tree.
@ -146,17 +161,23 @@ def scrape_files(repo, treehash, filelist, dest=""):
@param dest Destination path for files. Files will get retrieved with full
path from the repository, and the folder structure will get
created below dest as necessary.
@return Destination path.
@param timestamp_files List of files to also get the last modified date.
WARNING: this is SLOW!
@return Destination path, filename:timestamp dict.
'''
print("Scraping files from repository")
if dest == "":
dest = tempfile.mkdtemp()
treeobjects = get_lstree(repo, treehash, filelist)
timestamps = {}
for obj in treeobjects:
get_object(repo, treeobjects[obj], os.path.join(dest.encode(), obj))
for f in timestamp_files:
if obj.find(f) == 0:
timestamps[obj] = get_file_timestamp(repo, treehash, obj)
return dest
return [dest, timestamps]
def archive_files(repo, treehash, filelist, basename, tmpfolder="",
@ -182,7 +203,7 @@ def archive_files(repo, treehash, filelist, basename, tmpfolder="",
else:
temp_remove = False
workfolder = scrape_files(repo, treehash, filelist,
os.path.join(tmpfolder, basename))
os.path.join(tmpfolder, basename))[0]
if basename is "":
return ""
print("Archiving files from repository")
@ -192,11 +213,13 @@ def archive_files(repo, treehash, filelist, basename, tmpfolder="",
os.path.join(os.getcwd(), basename + ".7z"), basename],
cwd=tmpfolder, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output.communicate()
else:
elif archive == "tbz":
outfile = basename + ".tar.bz2"
tf = tarfile.open(outfile, "w:bz2")
tf.add(workfolder, basename)
tf.close()
else:
print("Files not archived")
if tmpfolder != workfolder:
shutil.rmtree(workfolder)
if temp_remove: