Cleansing Package

Inheritance

Inheritance diagram of seed.cleansing.models

Submodules

Models

class seed.cleansing.models.Cleansing(organization, *args, **kwargs)

Bases: object

ASSESSOR_FIELDS = [{'sort_column': 'pm_property_id', 'title': 'PM Property ID'}, {'sort_column': 'tax_lot_id', 'title': 'Tax Lot ID'}, {'sort_column': 'custom_id_1', 'title': 'Custom ID 1'}, {'sort_column': 'property_name', 'title': 'Property Name'}, {'sort_column': 'address_line_1', 'title': 'Address Line 1'}, {'sort_column': 'address_line_2', 'title': 'Address Line 2'}, {'sort_column': 'district', 'title': 'County/District/Ward/Borough'}, {'sort_column': 'lot_number', 'title': 'Lot Number'}, {'sort_column': 'block_number', 'title': 'Block Number'}, {'sort_column': 'city', 'title': 'City'}, {'sort_column': 'state_province', 'title': 'State Province'}, {'sort_column': 'postal_code', 'title': 'Postal Code'}, {'sort_column': 'year_built', 'title': 'Year Built'}, {'sort_column': 'use_description', 'title': 'Use Description'}, {'sort_column': 'building_count', 'title': 'Building Count'}, {'sort_column': 'property_notes', 'title': 'Property Notes'}, {'sort_column': 'recent_sale_date', 'title': 'Recent Sale Date'}, {'sort_column': 'owner', 'title': 'Owner'}, {'sort_column': 'owner_address', 'title': 'Owner Address'}, {'sort_column': 'owner_city_state', 'title': 'Owner City'}, {'sort_column': 'owner_postal_code', 'title': 'Owner Postal Code'}, {'sort_column': 'owner_email', 'title': 'Owner Email'}, {'sort_column': 'owner_telephone', 'title': 'Owner Telephone'}, {'sort_column': 'gross_floor_area', 'title': 'Gross Floor Area'}, {'sort_column': 'energy_score', 'title': 'Energy Score'}, {'sort_column': 'site_eui', 'title': 'Site EUI'}, {'sort_column': 'generation_date', 'title': 'Generation Date'}, {'sort_column': 'release_date', 'title': 'Release Date'}, {'sort_column': 'year_ending', 'title': 'Year Ending'}, {'sort_column': 'created', 'title': 'Creation Date'}, {'sort_column': 'modified', 'title': 'Modified Date'}, {'sort_column': 'conditioned_floor_area', 'title': 'Conditioned Floor Area'}, {'sort_column': 'occupied_floor_area', 'title': 'Occupied Floor Area'}, {'sort_column': 'site_eui_weather_normalized', 'title': 'Site EUI Weather Normalized'}, {'sort_column': 'source_eui', 'title': 'Source EUI'}, {'sort_column': 'source_eui_weather_normalized', 'title': 'Source EUI Weather Normalized'}, {'sort_column': 'building_certification', 'title': 'Building Certification'}, {'sort_column': 'energy_alerts', 'title': 'Energy Alerts'}, {'sort_column': 'space_alerts', 'title': 'Space Alerts'}]
ASSESSOR_FIELDS_BY_COLUMN = {'lot_number': {'sort_column': 'lot_number', 'title': 'Lot Number'}, 'owner_address': {'sort_column': 'owner_address', 'title': 'Owner Address'}, 'owner_postal_code': {'sort_column': 'owner_postal_code', 'title': 'Owner Postal Code'}, 'block_number': {'sort_column': 'block_number', 'title': 'Block Number'}, 'source_eui_weather_normalized': {'sort_column': 'source_eui_weather_normalized', 'title': 'Source EUI Weather Normalized'}, 'owner_email': {'sort_column': 'owner_email', 'title': 'Owner Email'}, 'modified': {'sort_column': 'modified', 'title': 'Modified Date'}, 'year_ending': {'sort_column': 'year_ending', 'title': 'Year Ending'}, 'building_count': {'sort_column': 'building_count', 'title': 'Building Count'}, 'postal_code': {'sort_column': 'postal_code', 'title': 'Postal Code'}, 'owner': {'sort_column': 'owner', 'title': 'Owner'}, 'property_name': {'sort_column': 'property_name', 'title': 'Property Name'}, 'source_eui': {'sort_column': 'source_eui', 'title': 'Source EUI'}, 'custom_id_1': {'sort_column': 'custom_id_1', 'title': 'Custom ID 1'}, 'city': {'sort_column': 'city', 'title': 'City'}, 'property_notes': {'sort_column': 'property_notes', 'title': 'Property Notes'}, 'district': {'sort_column': 'district', 'title': 'County/District/Ward/Borough'}, 'conditioned_floor_area': {'sort_column': 'conditioned_floor_area', 'title': 'Conditioned Floor Area'}, 'occupied_floor_area': {'sort_column': 'occupied_floor_area', 'title': 'Occupied Floor Area'}, 'generation_date': {'sort_column': 'generation_date', 'title': 'Generation Date'}, 'energy_alerts': {'sort_column': 'energy_alerts', 'title': 'Energy Alerts'}, 'space_alerts': {'sort_column': 'space_alerts', 'title': 'Space Alerts'}, 'pm_property_id': {'sort_column': 'pm_property_id', 'title': 'PM Property ID'}, 'use_description': {'sort_column': 'use_description', 'title': 'Use Description'}, 'site_eui': {'sort_column': 'site_eui', 'title': 'Site EUI'}, 'site_eui_weather_normalized': {'sort_column': 'site_eui_weather_normalized', 'title': 'Site EUI Weather Normalized'}, 'building_certification': {'sort_column': 'building_certification', 'title': 'Building Certification'}, 'energy_score': {'sort_column': 'energy_score', 'title': 'Energy Score'}, 'state_province': {'sort_column': 'state_province', 'title': 'State Province'}, 'year_built': {'sort_column': 'year_built', 'title': 'Year Built'}, 'created': {'sort_column': 'created', 'title': 'Creation Date'}, 'release_date': {'sort_column': 'release_date', 'title': 'Release Date'}, 'gross_floor_area': {'sort_column': 'gross_floor_area', 'title': 'Gross Floor Area'}, 'owner_city_state': {'sort_column': 'owner_city_state', 'title': 'Owner City'}, 'owner_telephone': {'sort_column': 'owner_telephone', 'title': 'Owner Telephone'}, 'recent_sale_date': {'sort_column': 'recent_sale_date', 'title': 'Recent Sale Date'}, 'tax_lot_id': {'sort_column': 'tax_lot_id', 'title': 'Tax Lot ID'}, 'address_line_2': {'sort_column': 'address_line_2', 'title': 'Address Line 2'}, 'address_line_1': {'sort_column': 'address_line_1', 'title': 'Address Line 1'}}
static cache_key(file_pk)

Static method to return the location of the cleansing results from redis.

Parameters:file_pk – Import file primary key
Returns:
cleanse(data)

Send in data as a queryset from the BuildingSnapshot ids.

Parameters:data – rows of data to be cleansed
Returns:
data_type_check(datum)

Check the data types of the fields. These should never be wrong as these are the data in the database.

This chunk of code is currently ignored.

Parameters:datum – Database record containing the BS version of the fields populated
Returns:None
in_range_checking(datum)

Check for errors in the min/max of the values.

Parameters:datum – Database record containing the BS version of the fields populated
Returns:None
static initialize_cache(file_pk)

Initialize the cache for storing the results. This is called before the celery tasks are chunked up.

Parameters:file_pk – Import file primary key
Returns:
missing_matching_field(datum)

Look for fields in the database that are not matched. Missing is defined as a None in the database

Parameters:datum – Database record containing the BS version of the fields populated
Returns:None

# TODO: NL: Should we check the extra_data field for the data?

missing_values(datum)

Look for fields in the database that are empty. Need to know the list of fields that are part of the cleansing section.

The original intent of this method would be very intensive to run (looking at all fields except the ignored). This method was changed to check for required values.

Parameters:datum – Database record containing the BS version of the fields populated
Returns:None

# TODO: Check the extra_data field for the data?

prune_data()

Prune the results will remove any entries that have zero cleansing_results

Returns:None
reset_results()
save_to_cache(file_pk)

Save the results to the cache database. The data in the cache are stored as a list of dictionaries. The data in this class are stored as a dict of dict. This is important to remember because the data from the cache cannot be simply loaded into the above structure.

Parameters:file_pk – Import file primary key
Returns:None
class seed.cleansing.models.Rules(id, org, field, enabled, category, type, min, max, severity, units)

Bases: django.db.models.base.Model

exception DoesNotExist

Bases: django.core.exceptions.ObjectDoesNotExist

exception Rules.MultipleObjectsReturned

Bases: django.core.exceptions.MultipleObjectsReturned

static Rules.delete_rules(organization)
Rules.get_category_display(*moreargs, **morekwargs)
Rules.get_severity_display(*moreargs, **morekwargs)
Rules.get_type_display(*moreargs, **morekwargs)
static Rules.initialize_rules(organization)
Rules.objects = <django.db.models.manager.Manager object>
Rules.org

Accessor to the related object on the forward side of a many-to-one or one-to-one relation.

In the example:

class Child(Model):
    parent = ForeignKey(Parent, related_name='children')

child.parent is a ForwardManyToOneDescriptor instance.

static Rules.restore_defaults(organization)

Tasks

Tests

class seed.cleansing.tests.CleansingDataSample(methodName='runTest')

Bases: django.test.testcases.TestCase

setUp()
test_cleanse()
class seed.cleansing.tests.CleansingDataTestCoveredBuilding(methodName='runTest')

Bases: django.test.testcases.TestCase

setUp()
test_cleanse()
test_simple_login()
class seed.cleansing.tests.CleansingDataTestPM(methodName='runTest')

Bases: django.test.testcases.TestCase

setUp()
test_cleanse()
class seed.cleansing.tests.CleansingViewTests(methodName='runTest')

Bases: django.test.testcases.TestCase

setUp()
test_get_cleansing_results()
test_get_csv()
test_get_progress()

URLs

Views

seed.cleansing.views.get_cleansing_results(request, *args, **kwargs)

Retrieve the details of the cleansing script.

seed.cleansing.views.get_csv(request, *args, **kwargs)

Download a csv of the results.

seed.cleansing.views.get_progress(request, *args, **kwargs)

Return the progress of the cleansing.