Keryx

Merge lp://staging/keryx/unstable into lp://staging/keryx/stable

  1. unstable
  2. Merge into stable
Proposed by mac9416
Status: Merged
Merged at revision: not available
Proposed branch: lp://staging/keryx/unstable
Merge into: lp://staging/keryx/stable
Diff against target: None lines
To merge this branch: bzr merge lp://staging/keryx/unstable
Reviewer Review Type Date Requested Status
Chris Oliver Pending
Review via email: mp+9720@code.staging.launchpad.net
To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file 'doc/LocalRepo-Add.py'
2--- doc/LocalRepo-Add.py 1970-01-01 00:00:00 +0000
3+++ doc/LocalRepo-Add.py 2009-08-05 20:27:15 +0000
4@@ -0,0 +1,89 @@
5+import os, os.path, shutil
6+from gzip import GzipFile
7+
8+REPO_NAME = 'quick_repo'
9+repodir = os.path.join(os.getcwd(), REPO_NAME)
10+
11+listdir = os.path.join(os.getcwd(), 'lists')
12+packdir = os.path.join(os.getcwd(), 'packages')
13+if not os.path.exists(listdir):
14+ print "There is somethin' wrong with you, son! You don't even have a ./lists directory. Make sure that you are running this script from within an APT-type Keryx project.'"
15+if not os.path.exists(packdir):
16+ print "What's this?! I can't find a ./packages directory. Make sure that you are running this script from within an APT-type Keryx project."
17+listfiles = os.listdir(listdir)
18+packfiles = os.listdir(packdir)
19+
20+def stripDotCom(text):
21+ append = False
22+ final = ''
23+ for item in text.split('_'):
24+ if item == 'dists': append = True
25+ if append:
26+ final = os.path.join(final, item)
27+ return final
28+
29+def convertUnderscores(text, prefix): # Returns a file that has converted underscores ("_") to / or \ in a filename.
30+ final = ''
31+ for item in text.split('_'):
32+ final = os.path.join(final, item)
33+ return os.path.join(prefix, final)
34+
35+def splitPacks(text):
36+ filename = ''
37+ packs = {}
38+ for block in text.split('\n\n'):
39+ for line in block.split('\n'):
40+ if line.startswith('Filename: '):
41+ filename = line[10:]
42+ if filename != '':
43+ packs.update({filename:block})
44+ return packs
45+
46+count = 0
47+lists = {}
48+for filename in listfiles:
49+ try:
50+ listfile = open(os.path.join(listdir, filename), 'rb')
51+ except:
52+ print "Well, that list just wouldn't load: " + filename
53+ continue
54+ packs = splitPacks(listfile.read()) # splitPacks returns {packfilename:packtext, etc.}
55+ listfile.close()
56+ if lists.has_key(stripDotCom(filename)): # If a list located in the same part of the repo has already been scanned...
57+ lists[stripDotCom(filename)].update(packs) # Simply add the current data to that file.
58+ else: # Else...
59+ lists.update({stripDotCom(filename):packs})# Add the new list!
60+ countdup = len(lists[stripDotCom(filename)]) - len(packs)
61+ count += 1
62+ print "Loaded", count, "of", len(listfiles), "lists ->", str(len(packs)), "more packages,", str(countdup), "duplicates."
63+
64+for packlist in lists.iteritems():
65+ packlisttext = ""
66+ dirlist = os.path.abspath(os.path.join(repodir, packlist[0]))
67+ for pack in packlist[1].iteritems():
68+ dirpack = os.path.abspath(os.path.join(repodir, pack[0]))
69+ if os.path.split(pack[0])[-1] in packfiles and not os.path.exists(dirpack): # If the file from the index file is in the packages directory but not yet in repo...
70+ packlisttext += (pack[1] + '\n\n') # Add this to the new index file,
71+ if not os.path.exists(os.path.dirname(dirpack)): # Then copy the deb into the repo.
72+ try:
73+ os.makedirs(os.path.dirname(dirpack)) # If the destination dir doesn't exist, create it.
74+ print "Creating dir: " + os.path.dirname(pack[0])
75+ except:
76+ print "Failed creating dir: " + os.path.dirname(pack[0])
77+ pass
78+ print "Copying: " + os.path.split(pack[0])[-1] + "..."
79+ shutil.copy(os.path.join(packdir, os.path.split(pack[0])[-1]), dirpack)
80+ if packlisttext != '': # Only bother with the Packages.gz file if there is a reason
81+ if not os.path.exists(os.path.dirname(dirlist)):
82+ try:
83+ os.makedirs(os.path.dirname(dirlist))
84+ print "Creating dir: " + os.path.dirname(packlist[0])
85+ except:
86+ print "Failed creating dir: " + os.path.dirname(packlist[0])
87+ pass
88+ print "Writing file: " + packlist[0] + '.gz'
89+ packlistfile = file(dirlist + '.gz', 'ab') # If repo already has this Packages.gz file then add the new files to it.
90+ gzfile = GzipFile(dirlist, 'ab', 9, packlistfile)
91+ gzfile.write(packlisttext)
92+ gzfile.close()
93+ packlistfile.close()
94
95=== added file 'doc/VerifyChecksums.py'
96--- doc/VerifyChecksums.py 1970-01-01 00:00:00 +0000
97+++ doc/VerifyChecksums.py 2009-08-05 20:27:15 +0000
98@@ -0,0 +1,86 @@
99+# original code from mac9416's script 'QuickRepo.py'
100+# modified and repurposed by jaseen
101+
102+import os, os.path, hashlib
103+#from gzip import GzipFile
104+
105+
106+listdir = os.path.join(os.getcwd(), 'lists')
107+packdir = os.path.join(os.getcwd(), 'packages')
108+hashtype = ('MD5sum: ', 'md5')
109+#hashtype = ('SHA1: ', 'sha1') # sha1 checks keep failing completely !?! Use md5
110+
111+if not os.path.exists(listdir):
112+ print "There is somethin' wrong with you, son! You don't even have a ./lists directory. Make sure that you are running this script from within an APT-type Keryx project.'"
113+if not os.path.exists(packdir):
114+ print "What's this?! I can't find a ./packages directory. Make sure that you are running this script from within an APT-type Keryx project."
115+listfiles = os.listdir(listdir)
116+packfiles = os.listdir(packdir)
117+
118+def calcChecksum(filepath):
119+ try: # This code originally found in doubledetector.py from http://sebsauvage.net/python/
120+ file = open(filepath,'rb') # some modifications made of course.
121+ if hashtype[1] == 'md5':
122+ digest = hashlib.md5()
123+ elif hashtype[1] == 'sha1':
124+ digest = hashlib.sha1()
125+ else:
126+ return '0'
127+ data = file.read(65536)
128+ while len(data) != 0:
129+ digest.update(data)
130+ data = file.read(65536)
131+ file.close()
132+ except:
133+ return '0'
134+ else:
135+ return digest.hexdigest()
136+
137+
138+def ParseLists(text):
139+ filename = ''
140+ hashsum = ''
141+ packs = {}
142+ for block in text.split('\n\n'):
143+ for line in block.split('\n'):
144+ if line.startswith('Filename: '):
145+ filename = os.path.split(line[10:])[-1]
146+ if line.startswith(hashtype[0]):
147+ hashsum = line[8:]
148+ if filename != '':
149+ if hashsum == '':
150+ print "A file without a '" + hashtype[0] + "', interesting: " + filename
151+ else:
152+ packs.update({filename:hashsum})
153+ return packs
154+
155+count = 0
156+lists = {}
157+for filename in listfiles:
158+ try:
159+ listfile = open(os.path.join(listdir, filename), 'rb')
160+ except:
161+ print "Well, that list just wouldn't load: " + filename
162+ continue
163+ packs = ParseLists(listfile.read()) # ParseLists returns {packfilename:packmd5sum, etc.}
164+ listfile.close()
165+ lists.update(packs) # Add the new list!
166+ count += 1
167+ print count, "read of", len(listfiles), "-", len(packs), "more names, ", len(lists), "unique"
168+
169+packlistsums = 0
170+failed = 0
171+nocalc = 0
172+listsorted = sorted(lists.iterkeys())
173+for key in listsorted:
174+ if key in packfiles: # If the file from the index file is in the packages directory...
175+ sum = calcChecksum(os.path.join(packdir, key))
176+ packlistsums += 1
177+ if sum != lists[key] and sum != '0':
178+ os.remove(os.path.join(packdir, key))
179+ print "Failed " + str(key) + ": Removed."
180+ failed += 1
181+ elif sum == '0':
182+ print "Could not calc checksum", key
183+ nocalc += 1
184+print "Of", packlistsums, "debs processed,", failed, "failed,", nocalc, "could not be checked."

Subscribers

People subscribed via source and target branches