source: ipk/ipkg-utils-050831/upload-package.cgi@ 4860

Last change on this file since 4860 was 4860, checked in by obi, 15 years ago

[ipkg] fix

File size: 19.4 KB
Line 
1#!/usr/bin/python
2
3import sys, os, cgi, commands, time
4import re
5import posixfile
6import ConfigParser
7import ipkg
8import smtplib
9import libxml2
10import newtempfile
11import string
12
13config = '/etc/packman.conf'
14
15
16feeds_base = '/home/jamey/feeds'
17feeds_base = '/home/ftp/feeds'
18cgi.logfile = '/tmp/upload'
19tmp_dir = '/tmp'
20ipkgfind_url = 'http://ipkgfind.handhelds.org'
21
22update_package_list = 0
23cp = ConfigParser.ConfigParser()
24cp.read(config)
25try:
26 if cp.has_option('config', 'feeds_base'):
27 feeds_base = cp.get('config', 'feeds_base')
28 if cp.has_option('config', 'logfile'):
29 cgi.logfile = cp.get('config', 'logfile')
30 if cp.has_option('config', 'tmp_dir'):
31 tmp_dir = cp.get('config', 'tmp_dir')
32 if cp.has_option('config', 'update_index'):
33 update_package_list = cp.get('config', 'update_index')
34except:
35 pass
36cp = None
37
38logprefix = time.strftime('%c') + ' -- '
39try:
40 for k in ['REMOTE_ADDR']:
41 if (os.environ.has_key(k)):
42 logprefix = logprefix + ('%s' % (os.environ[k],))
43except:
44 pass
45
46def log(string):
47 cgi.log(logprefix + ': ' + string)
48
49def filename_is_valid(filename):
50 if re.match(r'^[+-._A-Za-z0-9]+$', filename): return 1
51 return 0
52
53def copy(srcfilename, dstfilename):
54 srcfile = open(srcfilename, 'r')
55 dstfile = open(dstfilename, 'w')
56 if not srcfile:
57 return 1
58 if not dstfile:
59 return 1
60 while (1):
61 str = srcfile.read(512)
62 if (str):
63 nbytes = dstfile.write(str)
64 else:
65 break
66 dstfile.close()
67 srcfile.close()
68 return 0
69
70def make_index(feed_dir):
71 os.chdir(feed_dir)
72 f = posixfile.open(tmp_dir + "/Packages.lck", 'w')
73 f.lock('w|')
74 (rc, outtext) = commands.getstatusoutput('/usr/local/bin/ipkg-make-index . > /tmp/Packages')
75 if (rc != 0):
76 print ('<b>Failed to create Packages file with error=%s and output=%s</b>' % (rc,outtext))
77 else:
78 commands.getstatusoutput('mv /tmp/Packages Packages')
79 f.lock('u')
80 f.close()
81 print '<b>Updated Packages</b>'
82 return
83
84def update_index(feed_dir, pkgfilename):
85 f = posixfile.open(tmp_dir + "/Packages.lck", 'w')
86 f.lock('w|')
87 (rc, outtext) = commands.getstatusoutput('ipkg-update-index %s %s' % (feed_dir, pkgfilename) )
88 if (rc != 0):
89 print ('<b>Failed to update Packages file with error=%s and output=%s</b>' % (rc,outtext))
90 f.lock('u')
91 f.close()
92 print ('<b>added %s to Packages</b>' % (pkgfilename,))
93 return
94
95def announce_to_cia(uploader, feedname, filename, changenotice=None):
96 msg = "From: commits@handhelds.org\r\nTo: commits@picogui.org\r\nContent-Type: text/plain;\r\nSubject: SendToChannels handhelds.org\r\n\r\n{light blue}" + filename +"{normal} uploaded to {light green}" + feedname + "{normal} by {orange}" + uploader + "{normal}\r\n"
97 mailmsg = "From: " + uploader + "\r\nTo: familiar-updates@handhelds.org\r\nContent-Type: text/plain;\r\nSubject: " + filename + " uploaded to " + feedname + "\r\n\r\n\r\n"
98 if changenotice:
99 mailmsg = mailmsg + changenotice + "\r\n"
100 server = smtplib.SMTP('localhost')
101 server.sendmail('commits@handhelds.org', 'commits@picogui.org', msg)
102 server.sendmail(uploader, 'familiar-updates@handhelds.org', mailmsg)
103 server.quit()
104
105def fixup_filename(filename):
106 if filename and filename[:8] == "C:\\Temp\\":
107 filename = filename[8:]
108 return filename
109
110def update_feed_rss(uploader, feedname, filename, upload_time):
111 rss_filename = feeds_base + '/' + feedname + '/upload.rss'
112 m = re.match(r'([^_]+)_([^_]+)_([^_]+).ipk', filename)
113 if not m:
114 return
115 (pkg_name, pkg_version, pkg_arch) = m.groups()
116 doc = libxml2.parseFile(rss_filename)
117 items = []
118 c = doc.children.children
119 while c:
120 if c.name == 'item':
121 items.append(c)
122 c = c.next
123 oldest = items[len(items)-1]
124 items_parent = items[0].parent
125 c = oldest.children
126 while c:
127 if c.name == 'title':
128 c.children.setContent('%s (version %s arch %s by %s on %s)' % (pkg_name, pkg_version, pkg_arch, uploader, upload_time))
129 elif c.name == 'link':
130 c.children.setContent(ipkgfind_url + '/details.phtml?package=' + pkg_name)
131 else:
132 pass
133 c = c.next
134 # now move c from end of list to head of list
135 oldest.unlinkNode()
136 items_parent.children.addPrevSibling(oldest)
137 f = open(rss_filename, 'w')
138 doc.dump(f)
139
140def handle_batch_upload(feeditem, batchitem, tarfileitem, sigfileitem, changenotice=None):
141 global logprefix
142 global update_package_list
143
144 feedname = feeditem.value
145 sigfilename = sigfileitem.filename
146 batchfilename = batchitem.filename
147 sigfilename = fixup_filename(sigfilename)
148 batchfilename = fixup_filename(batchfilename)
149
150 if not filename_is_valid(feedname):
151 print ('<b>%s</b> is an invalid filename\n' % (feedname,))
152 log('invalid feedname %s' % (feedname,))
153 return 0
154
155 feed_dir = feeds_base + '/' + feedname + '/'
156
157 if not filename_is_valid(batchfilename):
158 print ('<b>%s</b> is an invalid filename\n' % (batchfilename,))
159 log('invalid filename %s' % (batchfilename,))
160 return 0
161 if not filename_is_valid(sigfilename):
162 print ('<b>%s</b> is an invalid filename\n' % (sigfilename,))
163 log('invalid sigfilename %s' % (sigfilename,))
164 return 0
165 if (batchfilename == 'keyring.gpg' or sigfilename == 'keyring.gpg'):
166 print '<b>not allowed to overwrite keyring</b>'
167 return 0
168 if (os.path.exists(feed_dir + '/' + batchfilename)):
169 print ('<b>%s</b> already exists in feed directory' % (batchfilename,))
170 return 0
171 if (os.path.exists(feed_dir + '/' + sigfilename)):
172 print ('<b>%s</b> already exists in feed directory' % (sigfilename,))
173 return 0
174
175 files = []
176
177 for line in string.split(batchitem.value, "\n"):
178 fields = string.split(line)
179 if not fields:
180 continue
181 md5sum = fields[0]
182 filename = fields[1]
183 if not filename_is_valid(filename):
184 print ('<b>%s</b> is an invalid filename' % (filename,))
185 return 0
186 if (filename == 'keyring.gpg' or sigfilename == 'keyring.gpg'):
187 print '<b>not allowed to overwrite keyring</b>'
188 return 0
189 if (os.path.exists(feed_dir + '/' + filename)):
190 print ('<b>%s</b> already exists in feed directory' % (filename,))
191 return 0
192 files.append(filename)
193
194 tmpdir = newtempfile.mkdtemp();
195
196 try:
197 batchfile = open(tmpdir + "/" + batchfilename, 'w')
198 batchfile.write(batchitem.value)
199 batchfile.close()
200
201 sigfile = open(tmpdir + "/" + sigfilename, 'w')
202 sigfile.write(sigfileitem.value)
203 sigfile.close()
204
205 keyringfile = feed_dir + 'keyring.gpg'
206 try:
207 (exitstatus, outtext) = commands.getstatusoutput("/usr/bin/gpgv --keyring %s %s" % (keyringfile, tmpdir + '/' + sigfilename))
208 except:
209 cgi.log('caught an exception in gpg')
210 raise
211 print '<pre>' + outtext + '</pre>'
212 if (exitstatus != 0):
213 log('/usr/bin/gpgv %s %s exitstatus=%d' % (filename, sigfilename, exitstatus,))
214 log(' ' + outtext)
215 rc = exitstatus
216 raise "failed"
217
218 match = re.search("Good signature from.*\<(.+)\>", outtext)
219 if (match):
220 uploader = match.group(1)
221 else:
222 log('Could not extract uploader name from gpg output:')
223 log(' ' + outtext)
224 print "Something went wrong parsing output from gpg!"
225 raise "failed"
226
227 tarcmd = "tar --directory=%s -f - -x %s" % (tmpdir, string.join(files, " "))
228
229 tarpipe, tarout, tarerr = os.popen3(tarcmd)
230 tarpipe.write(tarfileitem.value)
231 tarpipe.close()
232
233 while tarerr.readline() != "":
234 pass
235
236 tarout.close()
237 tarerr.close()
238
239 os.chdir(tmpdir)
240 md5sumcmd = "md5sum -c %s" % (batchfilename,)
241
242 try:
243 (exitstatus, outtext) = commands.getstatusoutput(md5sumcmd)
244 except:
245 print "md5sum threw an exception"
246 cgi.log('caught an exception in md5sum')
247 raise
248
249 if (exitstatus != 0):
250 print "<b>md5sum check failed</b>"
251 print outtext
252 raise "failed"
253
254 os.chdir("/")
255
256 for filename in files:
257 if (re.search(".*\.ipk$", filename)):
258 ipk = ipkg.Package(tmpdir + "/" + filename)
259 if not ipk:
260 print '<p>Unable to parse ' + filename + '</p>'
261 raise "failed"
262 if not ipk.source:
263 print '<p><b>Error:</b> ' + filename + ' lacks a Source field in its control file.</p>'
264 raise "failed"
265 if not ipk.isdeb:
266 print '<p><b>Error:</b> ' + filename + ' is an old-format archive. Please upgrade ipkg-build.</p>'
267 raise "failed"
268 if ipk.filename_header:
269 print '<p><b>Error:</b> ' + filename + ' has a Filename header in its control file.</p>'
270 raise "failed"
271
272 for filename in files:
273 rc = copy(tmpdir + "/" + filename, feed_dir + "/" + filename)
274 if (rc != 0):
275 print ('<p>error %s copying %s to %s' % (rc, tmpdir + "/" + filename, feed_dir + "/" + filename))
276 os.unlink(tmpdir + "/" + filename)
277
278 log('Uploaded %s to %s' % (filename, feed_dir))
279
280 if (re.search(".*\.ipk$", filename)):
281 announce_to_cia(uploader, feedname, filename)
282 update_feed_rss(uploader, feedname, filename, time.strftime('%c'))
283
284 copy(tmpdir + "/" + sigfilename, feed_dir + "/" + sigfilename)
285 copy(tmpdir + "/" + batchfilename, feed_dir + "/" + batchfilename)
286
287 except:
288 print "Caught an exception"
289 os.chdir("/")
290 os.system("rm -r " + tmpdir)
291 raise
292
293 os.system("rm -r " + tmpdir)
294
295 print "<p>Packages file now updated every 10 minutes by cron job</p>\n"
296
297 return 1
298
299def handle_upload(feeditem, fileitem, sigfileitem, srcfileitem, changenotice=None):
300 global logprefix
301 global update_package_list
302
303 feedname = feeditem.value
304 filename = fileitem.filename
305 sigfilename = sigfileitem.filename
306 srcfilename = srcfileitem.filename
307 filename = fixup_filename(filename)
308 sigfilename = fixup_filename(sigfilename)
309 srcfilename = fixup_filename(srcfilename)
310 # print "filename=%s\n" % (filename,)
311 # print "sigfilename=%s\n" % (sigfilename,)
312 # print "srcfilename=%s\n" % (srcfilename,)
313
314 if not filename_is_valid(feedname):
315 print ('<b>%s</b> is an invalid filename\n' % (feedname,))
316 log('invalid feedname %s' % (feedname,))
317 return 0
318 if not filename_is_valid(filename):
319 print ('<b>%s</b> is an invalid filename\n' % (filename,))
320 log('invalid filename %s' % (filename,))
321 return 0
322 if not filename_is_valid(sigfilename):
323 print ('<b>%s</b> is an invalid filename\n' % (sigfilename,))
324 log('invalid sigfilename %s' % (sigfilename,))
325 return 0
326 if srcfilename and not filename_is_valid(srcfilename):
327 print ('<b>%s</b> is an invalid filename\n' % (srcfilename,))
328 log('invalid srcfilename %s' % (srcfilename,))
329 return 0
330 if (filename == 'keyring.gpg' or sigfilename == 'keyring.gpg'):
331 print '<b>not allowed to overwrite keyring</b>'
332 return 0
333
334 feed_dir = feeds_base + '/' + feedname + '/'
335 tmp_dir = "/tmp/"
336 file = open(tmp_dir + filename, 'w')
337 file.write(fileitem.value)
338 file.close()
339 sigfile = open(tmp_dir + sigfilename, 'w')
340 sigfile.write(sigfileitem.value)
341 sigfile.close()
342
343 ipk = ipkg.Package(tmp_dir + filename)
344 if not ipk:
345 print '<p>Unable to parse ' + filename + '</p>'
346 return 0
347 if not srcfilename and not ipk.source:
348 print '<p><b>Error:</b> ' + filename + ' lacks a Source field in its control file.</p>'
349 return 0
350 if not ipk.isdeb:
351 print '<p><b>Error:</b> ' + filename + ' is an old-format archive. Please upgrade ipkg-build.</p>'
352 return 0
353 if ipk.filename_header:
354 print '<p><b>Error:</b> ' + filename + ' has a Filename header in its control file.</p>'
355 return 0
356
357 keyringfile = feed_dir + 'keyring.gpg'
358 try:
359 (exitstatus, outtext) = commands.getstatusoutput("/usr/bin/gpgv --keyring %s %s" % (keyringfile, tmp_dir + sigfilename))
360 except:
361 cgi.log('caught an exception')
362 return 0
363 print '<pre>' + outtext + '</pre>'
364 if (exitstatus != 0):
365 log('/usr/bin/gpgv %s %s exitstatus=%d' % (filename, sigfilename, exitstatus,))
366 log(' ' + outtext)
367 rc = exitstatus
368 if (rc != 0):
369 return 0
370
371 match = re.search("Good signature from.*\<(.+)\>", outtext)
372 if (match):
373 uploader = match.group(1)
374 announce_to_cia(uploader, feedname, filename)
375 update_feed_rss(uploader, feedname, filename, time.strftime('%c'))
376
377 rc = copy(tmp_dir + filename, feed_dir + filename)
378 if (rc != 0):
379 print ('<p>error %s copying %s to %s' % (rc, tmp_dir + filename, feed_dir + filename))
380 os.unlink(tmp_dir + filename)
381
382 rc = copy(tmp_dir + sigfilename, feed_dir + sigfilename)
383 if (rc != 0):
384 print ('<p>error %s copying %s to %s' % (rc, tmp_dir + sigfilename, feed_dir + sigfilename))
385 os.unlink(tmp_dir + sigfilename)
386 cgi.logfile = feed_dir + 'upload.log'
387 log('uploaded %s the package %s %s' % (feedname, filename, sigfilename))
388 if changenotice:
389 cgi.log('\t' + changenotice)
390 if srcfilename:
391 log('srcfile=' + srcfilename)
392 overridefile = open(feed_dir + filename + ".override", 'w')
393 overridefile.write('Source: ' + srcfilename + '\n')
394 overridefile.close()
395 srcfile= open(tmp_dir + srcfilename, 'w')
396 srcfile.write(srcfileitem.value)
397 srcfile.close()
398 log('srcfile written to /tmp')
399
400 rc = copy(tmp_dir + srcfilename, feed_dir + srcfilename)
401 log('copied srcfile to feed, rc=%s' % (rc,))
402 if (rc != 0):
403 print ('<p>error %s copying %s to %s' % (rc, tmp_dir + sigfilename, feed_dir + sigfilename))
404 os.unlink(tmp_dir + srcfilename)
405 log('uploaded %s the source file %s' % (feedname, srcfilename))
406
407 print "<p>Packages file now updated every 10 minutes by cron job</p>\n"
408 # remove the tmp files
409 return 1
410
411def main():
412 global update_package_list
413 print "Content-type: text/html\n"
414 form = cgi.FieldStorage()
415 print '<html>'
416 print '<head><title>Handhelds.Org Package Manager -- upload</title></head>'
417 print '<body><h1>Handhelds.Org Package Manager -- upload</h1>'
418 if form and form.has_key("batchfilename") and form.has_key("signaturefilename") and form.has_key("datafilename"):
419 print '<ul>'
420 print '<li>'
421 print form["feedname"].value
422 print '<li>'
423 print form["batchfilename"].filename
424 print '<li>'
425 print form["signaturefilename"].filename
426 print '</ul>'
427 if form.has_key('update_index'):
428 update_package_list = form["update_index"]
429 if form.has_key('changenotice'):
430 changenotice= form['changenotice']
431 else:
432 changenotice=None
433 if handle_batch_upload(form["feedname"], form["batchfilename"], form["datafilename"], form["signaturefilename"], changenotice):
434 print '<p><b>Upload completed successfully</b>'
435 print ('<p><a href="/feeds/%s/Packages">Updated Packages File</a>' % (form["feedname"].value, ))
436 else:
437 print '<p><b>Upload failed</b>'
438 elif form and form.has_key("filename") and form.has_key("signaturefilename") and form.has_key("sourcefilename"):
439 print '<ul>'
440 print '<li>'
441 print form["feedname"].value
442 print '<li>'
443 print form["filename"].filename
444 print '<li>'
445 print form["signaturefilename"].filename
446 print '<li>'
447 print form["sourcefilename"].filename
448 print '</ul>'
449 if form.has_key('update_index'):
450 update_package_list = form["update_index"]
451 if form.has_key('changenotice'):
452 changenotice= form['changenotice']
453 else:
454 changenotice=None
455 if handle_upload(form["feedname"], form["filename"], form["signaturefilename"],form["sourcefilename"], changenotice):
456 print '<p><b>Upload completed successfully</b>'
457 print ('<p><a href="/feeds/%s/Packages">Updated Packages File</a>' % (form["feedname"].value, ))
458 else:
459 print '<p><b>Upload failed</b>'
460 elif form and form.has_key("feedname") and form.has_key("update_index"):
461 # feed_dir = feeds_base + '/' + form["feedname"].value + '/'
462 # make_index(feed_dir)
463 print "Packages file now updated hourly by cron job at 11 after the hour\n"
464 else:
465 print """
466 <h1>Signing Packages</h1>
467 The following command will generate the file foo.ipk.asc, containing
468 an ASCII signature of the package foo.ipk:
469 <pre>
470 gpg -sb --armor foo.ipk
471 </pre>
472 <p>
473
474 See <a
475 href="/pipermail/familiar/2001-December/003955.html">Jamey's
476 announcement of this package upload utility</a> for some more
477 details.
478
479 <h1>Package to Upload</h1>
480 <form action="/cgi-bin/upload-package.cgi" method="POST" enctype="multipart/form-data">
481 <table>
482 <tr><td>Feed Name <td><select name="feedname">
483 <option selected>unstable</option>
484 <option>devel</option>
485 </select></tr>
486 <tr><td>Filename <td><input type="file" name="filename"></tr>
487 <tr><td>OpenPGP Signature Filename <td> <input type="file" name="signaturefilename"></tr>
488 <tr><td>Sources Filename - Optional <td> <input type="file" name="sourcefilename"></tr>
489 <tr><td>Change Notice - Optional <td> <input type="file" name="changenotice"></tr>
490 </table>
491 <p>Please make sure that your package's
492 control file has a Source field.
493 <p>The Source field is intended to allow automated retrieval of the source
494 package from which your package was built. If you are uploading a source
495 package along with your binary package, the Source field should contain
496 the name of the source package. Otherwise, it should contain one of the
497 following:
498 <ul>
499 <li>a single URL pointing to a source tarball
500 <li>multiple URLs pointing to the multiple files (.dsc, .orig.tar.gz, .diff.gz)
501 comprising a Debian source package
502 </ul>
503 <input type="submit">
504 </form>
505"""
506 print '</body>'
507 print '</html>'
508
509main()
510
511
Note: See TracBrowser for help on using the repository browser.