From 4e81e72ef1a91356e112300bb2daabf9962107be Mon Sep 17 00:00:00 2001 From: Jason Michalski Date: Tue, 20 Dec 2011 18:46:45 -0500 Subject: [PATCH 4/4] Removed the old launchpad bugtracker. --- mysite/customs/bugtrackers/__init__.py | 30 ---- mysite/customs/bugtrackers/launchpad.py | 248 ------------------------------- mysite/customs/tests.py | 198 ------------------------ 3 files changed, 0 insertions(+), 476 deletions(-) delete mode 100644 mysite/customs/bugtrackers/__init__.py delete mode 100644 mysite/customs/bugtrackers/launchpad.py diff --git a/mysite/customs/bugtrackers/__init__.py b/mysite/customs/bugtrackers/__init__.py deleted file mode 100644 index 4dc8bd3..0000000 --- a/mysite/customs/bugtrackers/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# This file is part of OpenHatch. - -##################################################### -########## THIS IS DEPRECATED ####################### -########## WE INTEND TO MOVE THIS CODE ############## -########## INTO A SUBCLASS OF BUGPARSER OR ########## -########## BUGIMPORTER. IT WILL GO AWAY WITHIN ###### -########## A MONTH OR SO. ########################### -########## INFO: http://lists.openhatch.org/pipermail/devel/2011-November/002466.html # - -# Copyright (C) 2009 OpenHatch, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# This is an extremely skeletal bug tracker-like class, -# used by the mysite.search.tests.BugCanRefreshItself test. -class BugTracker(object): - def refresh_one_bug(self, bug): - pass diff --git a/mysite/customs/bugtrackers/launchpad.py b/mysite/customs/bugtrackers/launchpad.py deleted file mode 100644 index c4f3a17..0000000 --- a/mysite/customs/bugtrackers/launchpad.py +++ /dev/null @@ -1,248 +0,0 @@ -# This file is part of OpenHatch. - -##################################################### -########## THIS IS DEPRECATED ####################### -########## WE INTEND TO MOVE THIS CODE ############## -########## INTO A SUBCLASS OF BUGPARSER OR ########## -########## BUGIMPORTER. IT WILL GO AWAY WITHIN ###### -########## A MONTH OR SO. ########################### -########## INFO: http://lists.openhatch.org/pipermail/devel/2011-November/002466.html # - -# Copyright (C) 2010 Jack Grigg -# Copyright (C) 2010 OpenHatch, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import datetime -import logging - -import mysite.customs.models -import mysite.search.models -from mysite.base.helpers import sanitize_wide_unicode - -# Initialize Launchpad scraper thing -ConnectBug, ConnectBugList, URLBugListFilter = (None, None, None) - -# Initialize data dumpers -import mysite.search.lpb2json - -################################################## -# Crawler functions for Launchpad - -#FIXME: Add only those of the following that actually use Launchpad for development. -#u'apache-mod-digest' : u'apache-mod-digest' , u'bws-upload' : u'BWS-Upload' , u'pyjunitxml' : u'pyjunitxml' , u'bzr-search' : u'bzr search plugin' , u'bzr-email' : u'bzr email commit hook' , u'check' : u'check' , u'libsyncml' : u'libsyncml' , u'config-manager' : u'config-manager' , u'testscenarios' : u'testscenarios' , u'liburl' : u'liburl' , u'liblockdir' : u'lockdir' , u'bzr-guess' : u'bzr-guess' , u'etap' : u'etap' , u'gforth' : u'Gforth' , u'bitten' : u'Bitten' , u'sqlobject' : u'SQLObject' , u'bzr-ping' : u'Ping plugin for Bazaar' , u'unittest-ext' : u'unittest-ext' , u'pytz' : u'pytz' , u'funkload' : u'FunkLoad' , u'slony-i' : u'Slony-I' , u'zoneinfo' : u'The tz Database' , u'py-radius' : u'py-radius' , u'pypi' : u'Python Package Index' , u'pybabel' : u'Python Babel' , u'feedvalidator' : u'Feed Validator' , u'sphinx' : u'Sphinx' , u'mammoth-replicator' : u'Mammoth Replicator' , u'dbapi-compliance' : u'Python DBAPI Compliance Tests' , u'wget' : u'wget' , u'redhatcluster' : u'Red Hat Cluster' , u'bugzilla' : u'Bugzilla' , u'grepmap' : u'grepmap' , u'live-f1' : u'Live F1' , u'libnih' : u'libnih' , u'hct' : u'HCT' , u'upstart' : u'upstart ' , u'module-init-tools' : u'module-init-tools' , u'ubuntu-seeds' : u'Ubuntu Seeds' , u'usplash' : u'usplash' , u'merge-o-matic' : u'Merge-o-Matic' , u'uds-intrepid' : u'UDS Intrepid' , u'watershed' : u'watershed' , u'udev-extras' : u'Udev extras' , u'sreadahead' : u'sreadahead' , u'pybootchartgui' : u'pybootchartgui' , u'bootchart-collector' : u'bootchart-collector' , u'bootchart' : u'bootchart' , u'ubiquity' : u'ubiquity' , u'man-db' : u'man-db'} - -def dump_data_from_project(project): - - yield serialized - -# Callback to handle an update to a single Launchpad bug update -def handle_launchpad_bug_update(project_name, canonical_bug_link, new_data): - """ - We're going to store a bug in our database. First we want to - check to see if we've stored in the DB a stale copy of the same bug. - - Side-effect: We create or update a bug in the database. - In particular, if we already have a bug for this (project, - canonical_bug_link) pair, we modify that instead of creating a - duplicate entry. - - Right now we do not store last_modified time stamps; no one has - yet figured out what good it would do us.""" - project, _ = mysite.search.models.Project.objects.get_or_create(name=project_name) - new_data['project'] = project - bug, created = mysite.search.models.Bug.all_bugs.get_or_create( - canonical_bug_link=canonical_bug_link, defaults=new_data) - # 'defaults' means data used for creating but not getting - if created: - return bug # nothing to do! - else: - # else, update the local copy of the bug - for key in new_data: - setattr(bug, key, new_data[key]) - bug.save() - return bug - -def clean_lp_data_dict(lp_data_dict): - """Input: A single datum as returned by launchpadbugs. - - Output: query_data, new_data - two dicts that represent - processed data from Launchpad. - We've divided the data from launchpad into bug-identifying data (query_data) - and the rest of the data we want to store (new_data). - """ - # These are the invariants for every bug: together (well, maybe the - # bug link is enough, hush) they uniquely identify the bug. - query_data = {} - query_data['canonical_bug_link'] = lp_data_dict['url'] - - # If the above is the "key" that we use to find or create the record, - # these are the "value", - new_data = {} - new_data['title'] = lp_data_dict['title'] - new_data['description'] = lp_data_dict['text'] - - new_data['importance'] = lp_data_dict['importance'] - if new_data['importance'] is None: - new_data['importance'] = 'Unknown' - - # create set of people who have commented or created the ticket - people_involved = set() - if lp_data_dict['reporter']: - people_involved.add(dict2hashable(lp_data_dict['reporter'])) - for comment in lp_data_dict['comments']: - people_involved.add(dict2hashable(comment['user'])) - - new_data['people_involved'] = len(people_involved) - - new_data['submitter_username'] = lp_data_dict['reporter']['lplogin'] - new_data['submitter_realname'] = unicode(lp_data_dict['reporter']['realname']) - - # Handle dates - new_data['last_touched'] = datetime.datetime(*lp_data_dict['date_updated'][:6]) - new_data['last_polled'] = datetime.datetime.now() - - # Look for bitesize tag - # If no 'tags', pass - if 'bitesize' in lp_data_dict.get('tags', []): - new_data['good_for_newcomers'] = True - - status = lp_data_dict.get('status', 'Unknown') - if not status: - status = 'Unknown' - new_data['status'] = status - if new_data['status'].lower() in ('fix released', 'fix committed'): - new_data['looks_closed'] = True - # else looks_closed will be False due to the Bug default - new_data['date_reported'] = datetime.datetime( - *lp_data_dict['date_reported'][:6]) - - # Strip any wide Unicode from the dict - new_data = sanitize_wide_unicode(new_data) - - return query_data, new_data - -def dict2hashable(d): - return tuple(d.items()) - -def grab_lp_bugs(lp_project, openhatch_project_name): - '''Input: The name of a Launchpad project and its corresponding - OpenHatch project name. - - Side effect: Loops over the available issues in that project and - updates the database with them.''' - for data_dict in dump_data_from_project(lp_project): - data_dict['project'] = openhatch_project_name - query_data, new_data = clean_lp_data_dict(data_dict) - handle_launchpad_bug_update( - project_name=openhatch_project_name, - canonical_bug_link=query_data['canonical_bug_link'], - new_data=new_data) - -################################################## -# Import functions for Launchpad - -def refresh_bugs_from_all_indexed_launchpad_projects(): - # Look, ma, a hard-coded table that maps to - # OpenHatch project names from Launchpad.net project names. - # TimBL would be proud. - lpproj2ohproj = { 'lxml': 'lxml', - 'do': 'GNOME-do', - 'gwibber': 'Gwibber', - 'keryx': 'Keryx', - 'pynagram': 'Pynagram', - 'hardwar': 'Hardwar', - } - for launchpad_project_name in lpproj2ohproj: - openhatch_project_name = lpproj2ohproj[launchpad_project_name] - import_bugs_from_one_project(launchpad_project_name, - openhatch_project_name) - -def import_bugs_from_one_project(launchpad_project_name, - openhatch_project_name): - logging.info("Looking at bugs %s on Launchpad" % launchpad_project_name) - url = "https://bugs.launchpad.net/%s/+bugs" % launchpad_project_name - bug_filter = URLBugListFilter() - # no filtering; dump everything - TextBugList = ConnectBugList("text") - l = TextBugList(bug_filter(url)) - # convert elements into Bug objects - for bug in l: - openhatch_bug_link = 'https://bugs.launchpad.net/bugs/%d' % ( - bug.bugnumber) - refresh_one_launchpad_bug( - canonical_bug_link=openhatch_bug_link, - openhatch_project_name=openhatch_project_name) - -def refresh_all_launchpad_bugs(): - logging.info("Refreshing all Launchpad bugs.") - all_lp_bugs = mysite.search.models.Bug.all_bugs.filter( - canonical_bug_link__startswith='https://bugs.launchpad.net/') - logging.info("All %d of them." % all_lp_bugs.count()) - for lp_bug in all_lp_bugs: - refresh_one_launchpad_bug( - canonical_bug_link=lp_bug.canonical_bug_link, - openhatch_project_name=None) - -def refresh_one_launchpad_bug(canonical_bug_link, - openhatch_project_name): - logging.info("Checking on %s..." % canonical_bug_link) - # Either we already have the bug... - try: - bug = mysite.search.models.Bug.all_bugs.get( - canonical_bug_link=canonical_bug_link) - # ...or we need to create it - except mysite.search.models.Bug.DoesNotExist: - bug = mysite.search.models.Bug() - bug.canonical_bug_link = canonical_bug_link - - if bug.data_is_more_fresh_than_one_day(): - return - - # Okay, so it's stale. Refresh the sucker. - logging.info("Refreshing %s." % canonical_bug_link) - - # Set the project, if necessary - if openhatch_project_name is None: - pass - else: - # Get or create the OpenHatch project - openhatch_project, _ = mysite.search.models.Project.objects.get_or_create(name=openhatch_project_name) - - if bug.project_id != openhatch_project.id: - bug.project = openhatch_project - - # FIXME: One day, look at the bug data to see what project to use. - # This code incorrectly assumes bugs don't migrate from one project - # to another. - prefix = 'https://bugs.launchpad.net/bugs/' - assert canonical_bug_link.startswith(prefix) - bug_id_str = canonical_bug_link.split(prefix, 1)[1] - bug_id = int(bug_id_str) - TextBug = ConnectBug("text") - tb = TextBug(bug_id) - data_dict = mysite.search.lpb2json.obj2serializable(tb) - _, new_data = clean_lp_data_dict(data_dict) - for key in new_data: - setattr(bug, key, new_data[key]) - bug.last_polled = datetime.datetime.utcnow() - bug.save() - -########################################################## -# Specific sub-classes for individual bug trackers -########################################################## - -# Oops, there aren't any in this file. -# For Launchpad, take a look at mysite/customs/management/commands/customs_daily_tasks.py - diff --git a/mysite/customs/tests.py b/mysite/customs/tests.py index 0fb680b..d54f40e 100644 --- a/mysite/customs/tests.py +++ b/mysite/customs/tests.py @@ -923,204 +923,6 @@ sample_launchpad_data_snapshot.return_value = [dict( date_reported=time.localtime(), title="Joi's Lab AFS",)] -@skipIf(True, "Disabling old-style Launchpad tests.") -class AutoCrawlTests(django.test.TestCase): - @mock.patch('mysite.customs.bugtrackers.launchpad.dump_data_from_project', - sample_launchpad_data_snapshot) - @mock.patch('mysite.search.tasks.PopulateProjectLanguageFromOhloh') - def testSearch(self, do_nothing): - # Verify that we can't find a bug with the right description - self.assertRaises(mysite.search.models.Bug.DoesNotExist, - mysite.search.models.Bug.all_bugs.get, - title="Joi's Lab AFS") - # Now get all the bugs about rose - mysite.customs.bugtrackers.launchpad.grab_lp_bugs(lp_project='rose', - openhatch_project_name= - u'rose.makesad.us') - # Now see, we have one! - b = mysite.search.models.Bug.all_bugs.get(title="Joi's Lab AFS") - self.assertEqual(b.project.name, u'rose.makesad.us') - # Ta-da. - return b - - def test_running_job_twice_does_update(self): - b = self.testSearch() - b.description = u'Eat more potato starch' - b.title = u'Yummy potato paste' - b.save() - - new_b = self.testSearch() - self.assertEqual(new_b.title, "Joi's Lab AFS") # bug title restored - # thanks to fresh import - -@skipIf(True, "Disabling old-style Launchpad tests.") -class LaunchpadImporterTests(django.test.TestCase): - - @mock.patch('mysite.search.tasks.PopulateProjectLanguageFromOhloh') - def test_lp_update_handler(self, do_nothing): - '''Test the Launchpad import handler with some fake data.''' - some_date = datetime.datetime(2009, 4, 1, 2, 2, 2) - query_data = dict(project='GNOME-Do', - canonical_bug_link='http://example.com/1') - new_data = dict(title='Title', status='Godforsaken', - description='Everything should be better', - importance='High', - people_involved=1000 * 1000, - submitter_username='yourmom', - submitter_realname='Your Mom', - date_reported=some_date, - last_touched=some_date, - last_polled=some_date) - - # Create the bug... - mysite.customs.bugtrackers.launchpad.handle_launchpad_bug_update( - project_name=query_data['project'], - canonical_bug_link=query_data['canonical_bug_link'], - new_data=new_data) - # Verify that the bug was stored. - bug = Bug.all_bugs.get(canonical_bug_link= - query_data['canonical_bug_link']) - for key in new_data: - self.assertEqual(getattr(bug, key), new_data[key]) - - # Now re-do the update, this time with more people involved - new_data['people_involved'] = 1000 * 1000 * 1000 - # pass the data in... - mysite.customs.bugtrackers.launchpad.handle_launchpad_bug_update( - project_name=query_data['project'], - canonical_bug_link=query_data['canonical_bug_link'], - new_data=new_data) - # Do a get; this will explode if there's more than one with the - # canonical_bug_link, so it tests duplicate finding. - bug = Bug.all_bugs.get(canonical_bug_link= - query_data[u'canonical_bug_link']) - - for key in new_data: - self.assertEqual(getattr(bug, key), new_data[key]) - - @mock.patch('mysite.search.tasks.PopulateProjectLanguageFromOhloh') - def test_lp_data_clean(self, do_nothing): - now_t = (2009, 4, 1, 5, 13, 2) # partial time tuple - now_d = datetime.datetime(2009, 4, 1, 5, 13, 2) - # NOTE: We do not test for time zone correctness. - sample_in = dict(project='GNOME-Do', url='http://example.com/1', - title='Title', text='Some long text', - importance=None, status='Ready for take-off', - comments=[{'user': { - 'lplogin': 'jones', 'realname': 'Jones'}}], - reporter={'lplogin': 'bob', 'realname': 'Bob'}, - date_reported=now_t, - date_updated=now_t, - ) - sample_out_query = dict(canonical_bug_link='http://example.com/1') - sample_out_data = dict(title='Title', description='Some long text', - importance='Unknown', status='Ready for take-off', - people_involved=2, submitter_realname='Bob', - submitter_username='bob', - date_reported=now_d, - last_touched=now_d) - out_q, out_d = mysite.customs.bugtrackers.launchpad.clean_lp_data_dict(sample_in) - self.assertEqual(sample_out_query, out_q) - # Make sure last_polled is at least in the same year - self.assertEqual(out_d['last_polled'].year, datetime.date.today().year) - del out_d['last_polled'] - self.assertEqual(sample_out_data, out_d) - - @mock.patch('mysite.search.tasks.PopulateProjectLanguageFromOhloh') - def test_lp_data_wide_utf8(self, do_nothing): - sample_in = dict(affects=u'Do', - assignee=u'', - bugnumber=657268, - comments=[{'date': [2010, 10, 9, 10, 24, 48, 5, 282, -1], - 'number': 1, - 'text': u'Above characters may need a shaw font to be viewed, such as Andagii\n(7k):\nhttp://svn.gna.org/viewcvs/*checkout*/wesnoth/trunk/fonts/Andagii.ttf', - 'user': {'lplogin': u'arcriley', - 'realname': u'Arc "warthog" Riley' - } - }], - date=[2010, 10, 9, 10, 22, 6, 5, 282, -1], - date_reported=[2010, 10, 9, 10, 22, 6, 5, 282, -1], - date_updated=[2010, 10, 9, 10, 24, 49, 5, 282, -1], - description=u'Gnome-Do crashes randomly... enter a few keys such as \U00010451\U0001047b\U00010465\n("term" with en@shaw locale) then backspace...', - duplicate_of=None, - duplicates=[], - importance=u'Undecided', - milestone=u'', - private=False, - reporter={'lplogin': u'arcriley', 'realname': u'Arc "warthog" Riley'}, - security=False, - sourcepackage=u'do', - status=u'New', - summary=u'crashes on wide utf-8 input', - tags=[], - text=u'Gnome-Do crashes randomly... enter a few keys such as \U00010451\U0001047b\U00010465\n("term" with en@shaw locale) then backspace...', - title=u'crashes on wide utf-8 input', - url=u'https://bugs.launchpad.net/bugs/657268' - ) - bug_dr = datetime.datetime(2010, 10, 9, 10, 22, 6) - bug_du = datetime.datetime(2010, 10, 9, 10, 24, 49) - # NOTE: We do not test for time zone correctness. - sample_out_query = dict(canonical_bug_link='https://bugs.launchpad.net/bugs/657268') - sample_out_data = dict(title=u'crashes on wide utf-8 input', - description=u'Gnome-Do crashes randomly... enter a few keys such as ���\n("term" with en@shaw locale) then backspace...', - importance=u'Undecided', - status=u'New', - people_involved=1, - submitter_realname=u'Arc "warthog" Riley', - submitter_username=u'arcriley', - date_reported=bug_dr, - last_touched=bug_du - ) - out_q, out_d = mysite.customs.bugtrackers.launchpad.clean_lp_data_dict(sample_in) - self.assertEqual(sample_out_query, out_q) - # Make sure last_polled is at least in the same year - self.assertEqual(out_d['last_polled'].year, datetime.date.today().year) - del out_d['last_polled'] - self.assertEqual(sample_out_data, out_d) - -@skipIf(True, "Disabling old-style Launchpad tests.") -class LaunchpadImporterMarksFixedBugsAsClosed(django.test.TestCase): - def test(self): - '''Start with a bug that is "Fix Released" - - Verify that we set looks_closed to True''' - # retry this with committed->released - lp_data_dict = {'project': '', - 'url': '', - 'title': '', - 'text': '', - 'status': 'Fix Committed', - 'importance': '', - 'reporter': {'lplogin': '', 'realname': ''}, - 'comments': '', - 'date_updated': datetime.datetime.now().timetuple(), - 'date_reported': datetime.datetime.now().timetuple()} - # maybe I could have done this with a defaultdict of str with - # just the non-str exceptions - query_data, new_data = mysite.customs.bugtrackers.launchpad.clean_lp_data_dict( - lp_data_dict) - self.assertTrue(new_data['looks_closed']) - - def test_with_status_missing(self): - '''Verify we do not explode if Launchpad gives us a bug with no Status - - Verify that we set looks_closed to True''' - # retry this with committed->released - lp_data_dict = {'project': '', - 'url': '', - 'title': '', - 'text': '', - 'importance': '', - 'reporter': {'lplogin': '', 'realname': ''}, - 'comments': '', - 'date_updated': datetime.datetime.now().timetuple(), - 'date_reported': datetime.datetime.now().timetuple()} - # maybe I could have done this with a defaultdict of str with - # just the non-str exceptions - query_data, new_data = mysite.customs.bugtrackers.launchpad.clean_lp_data_dict( - lp_data_dict) - self.assertEqual(new_data['status'], 'Unknown') - @skipIf(mysite.base.depends.lxml.html is None, "To run these tests, you must install lxml. See ADVANCED_INSTALLATION.mkd for more.") class ParseCiaMessage(django.test.TestCase): def test_with_ansi_codes(self): -- 1.7.5.4