diff --git a/src/fetchcode/package.py b/src/fetchcode/package.py index fdca3ebd..1fa389b0 100644 --- a/src/fetchcode/package.py +++ b/src/fetchcode/package.py @@ -394,6 +394,425 @@ class IpkgDirectoryListedSource(DirectoryListedSource): source_archive_regex = r"^(ipkg[-_])([\w.-]*)(_arm.ipk|.tar.gz)$" ignored_files_and_dir = [] + @classmethod + def get_package_info(cls, package_url): + # Since there will be no new releases of ipkg, it's better to + # store them in a dictionary rather than fetching them every time. + archives = { + "0.99.88": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.88.tar.gk", + "date": " 2003-08-08T03:03:00", + }, + "0.99.89": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.89.tar.gk", + "date": " 2003-08-21T10:02:00", + }, + "0.99.102": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.102.tar.gk", + "date": " 2003-11-10T09:58:00", + }, + "0.99.107": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.107.tar.gk", + "date": " 2004-01-12T06:08:00", + }, + "0.99.110": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.110.tar.gk", + "date": " 2004-01-18T15:52:00", + }, + "0.99.118": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.118.tar.gk", + "date": " 2004-03-29T11:25:00", + }, + "0.99.119": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.119.tar.gk", + "date": " 2004-04-06T12:14:00", + }, + "0.99.120": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.120.tar.gk", + "date": " 2004-04-07T12:50:00", + }, + "0.99.121": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.121.tar.gk", + "date": " 2004-05-03T09:58:00", + }, + "0.99.122-1": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.122-1.tar.gk", + "date": " 2004-05-14T12:51:00", + }, + "0.99.122-2": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.122-2.tar.gk", + "date": " 2004-05-16T05:02:00", + }, + "0.99.122-3": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.122-3.tar.gk", + "date": " 2004-05-16T05:49:00", + }, + "0.99.122": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.122.tar.gk", + "date": " 2004-05-10T12:13:00", + }, + "0.99.124": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.124.tar.gk", + "date": " 2004-05-21T11:29:00", + }, + "0.99.125": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.125.tar.gk", + "date": " 2004-06-05T09:58:00", + }, + "0.99.126": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.126.tar.gk", + "date": " 2004-06-15T11:51:00", + }, + "0.99.127": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.127.tar.gk", + "date": " 2004-07-20T11:50:00", + }, + "0.99.129": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.129.tar.gk", + "date": " 2004-09-01T12:15:00", + }, + "0.99.130": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.130.tar.gk", + "date": " 2004-09-11T11:27:00", + }, + "0.99.131": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.131.tar.gk", + "date": " 2004-09-30T10:05:00", + }, + "0.99.132": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.132.tar.gk", + "date": " 2004-10-17T10:26:00", + }, + "0.99.133": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.133.tar.gk", + "date": " 2004-11-18T09:49:00", + }, + "0.99.134": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.134.tar.gk", + "date": " 2005-01-05T14:42:00", + }, + "0.99.135": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.135.tar.gk", + "date": " 2005-01-06T02:59:00", + }, + "0.99.136": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.136.tar.gk", + "date": " 2005-01-10T11:00:00", + }, + "0.99.137": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.137.tar.gk", + "date": " 2005-01-10T12:11:00", + }, + "0.99.138": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.138.tar.gk", + "date": " 2005-01-18T12:19:00", + }, + "0.99.139": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.139.tar.gk", + "date": " 2005-02-05T11:40:00", + }, + "0.99.140": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.140.tar.gk", + "date": " 2005-02-05T13:16:00", + }, + "0.99.141": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.141.tar.gk", + "date": " 2005-02-06T12:16:00", + }, + "0.99.142": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.142.tar.gk", + "date": " 2005-02-07T12:51:00", + }, + "0.99.143": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.143.tar.gk", + "date": " 2005-02-20T09:26:00", + }, + "0.99.144": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.144.tar.gk", + "date": " 2005-02-22T12:52:00", + }, + "0.99.145": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.145.tar.gk", + "date": " 2005-03-14T11:59:00", + }, + "0.99.146": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.146.tar.gk", + "date": " 2005-03-28T10:02:00", + }, + "0.99.148": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.148.tar.gk", + "date": " 2005-04-10T11:26:00", + }, + "0.99.149": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.149.tar.gk", + "date": " 2005-05-13T10:41:00", + }, + "0.99.151": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.151.tar.gk", + "date": " 2005-06-16T09:39:00", + }, + "0.99.152": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.152.tar.gk", + "date": " 2005-07-06T03:38:00", + }, + "0.99.153": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.153.tar.gk", + "date": " 2005-07-29T11:02:00", + }, + "0.99.154": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.154.tar.gk", + "date": " 2005-09-16T10:41:00", + }, + "0.99.155": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.155.tar.gk", + "date": " 2006-01-04T11:55:00", + }, + "0.99.156": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.156.tar.gk", + "date": " 2006-01-14T09:56:00", + }, + "0.99.157": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.157.tar.gk", + "date": " 2006-01-22T10:30:00", + }, + "0.99.158": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.158.tar.gk", + "date": " 2006-02-02T11:47:00", + }, + "0.99.161": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.161.tar.gk", + "date": " 2006-04-19T10:15:00", + }, + "0.99.162": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.162.tar.gk", + "date": " 2006-05-29T23:02:00", + }, + "0.99.163": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.163.tar.gk", + "date": " 2006-05-29T23:40:00", + }, + "0.9-1": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-1_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-2": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-2_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-3": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-3_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-4": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-4_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-5": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-5_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-6": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-6_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-6a": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-6a_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-6b": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-6b_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-6c": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-6c_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-demo3": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-demo3_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-demo4": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-demo4_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-demo5": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-demo5_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-demo6": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-demo6_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-jeh1": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-jeh1_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-jeh3": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-jeh3_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-jeh4": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-jeh4_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-jeh5": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-jeh5_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-jeh6": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-jeh6_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.9-jeh7": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-jeh7_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.9-jeh8": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-jeh8_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.9-jehb": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-jehb_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.9-jehc": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9-jehc_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.9": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.9_arm.ipk", + "date": " 2001-12-07T03:11:00", + }, + "0.99.8": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.8_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.9-1": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.9-1_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.9-2": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.9-2_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.9-3": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.9-3_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.9": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.9_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.10-2": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.10-2_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.10": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.10_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.11": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.11_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.13": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.13_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.14": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.14_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.15": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.15_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.16": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.16_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.18": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.18_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.19": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.19_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.20": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.20_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.21": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.21_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.22": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.22_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.23": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.23_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.24": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.24_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.27": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.27_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.28-1": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.28-1_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.28": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.28_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.29": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.29_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.31": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.31_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.32": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.32_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.33": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.33_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + "0.99.34": { + "url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.34_arm.ipk", + "date": " 2002-08-19T04:23:00", + }, + } + + version = package_url.version + if version and version in archives: + archive = archives[version] + yield Package( + homepage_url=cls.source_url, + download_url=archive["url"], + release_date=archive["date"], + **package_url.to_dict(), + ) + + else: + for version, data in archives.items(): + purl = PackageURL(type="generic", name="ipkg", version=version) + yield Package( + homepage_url=cls.source_url, + download_url=data["url"], + release_date=data["date"], + **purl.to_dict(), + ) + class UtilLinuxDirectoryListedSource(DirectoryListedSource): source_url = "https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/" @@ -608,7 +1027,7 @@ def get_packages_from_listing(purl, source_archive_url, regex, ignored_files_and version = version.strip("v").strip() if not version: continue - + modified_time = file.modified date = datetime.utcfromtimestamp(time.mktime(modified_time))