Reduce big files

tkrajina · tkrajina · commit 5f63552234ac · 2013-06-08T21:27:22.000+02:00
diff --git a/srtm/__init__.py b/srtm/__init__.py
@@ -39,7 +39,7 @@ def get_default_srtm_dir():
 
     return result
 
-def get_data(local_srtm_dir=None):
+def get_data(local_srtm_dir=None, reduce_big_files=False):
     """
     Get the utility object for querying elevation data.
 
@@ -49,6 +49,11 @@ def get_data(local_srtm_dir=None):
     On first run -- all files url will be stored and for every next elevation 
     query if the SRTM file is not found in local_srtm_dir it will be retrieved 
     and saved.
+
+    If reduce_big_files is True files for north America will be reduced to the 
+    same size as the rest of the world. It may be used to save disk space 
+    (because otherwise the tipical USA file is 25 megabaytes comparet to 2-3 
+    megabytes for the rest).
     """
     if not local_srtm_dir:
         local_srtm_dir = get_default_srtm_dir()
@@ -71,10 +76,12 @@ def get_data(local_srtm_dir=None):
         srtm3_files = mod_retriever.retrieve_all_files_urls(SRTM3_URL)
 
         f = open(files_list_file_name, 'w')
-        f.write(mod_json.dumps({'srtm1': srtm1_files, 'srtm3': srtm3_files}, sort_keys=True, indent=4))
+        f.write(mod_json.dumps({'srtm1': srtm1_files, 'srtm3': srtm3_files},
+                               sort_keys=True, indent=4))
         f.close()
 
     assert srtm1_files
     assert srtm3_files
 
-    return mod_data.GeoElevationData(srtm1_files, srtm3_files, local_srtm_dir)
+    return mod_data.GeoElevationData(srtm1_files, srtm3_files, local_srtm_dir,
+                                     reduce_big_files=reduce_big_files)
diff --git a/srtm/data.py b/srtm/data.py
@@ -44,12 +44,15 @@ class GeoElevationData:
     # Lazy loaded files used in current app:
     files = None
 
-    def __init__(self, srtm1_files, srtm3_files, files_directory):
+    def __init__(self, srtm1_files, srtm3_files, files_directory,
+                 reduce_big_files=False):
         self.srtm1_files = srtm1_files
         self.srtm3_files = srtm3_files
         # Place where local files are stored:
         self.files_directory = files_directory
 
+        self.reduce_big_files = reduce_big_files
+
         self.files = {}
 
     def get_elevation(self, latitude, longitude, approximate=None):
@@ -124,12 +127,34 @@ def retrieve_or_load_file_data(self, file_name):
         if not data:
             return None
 
+        if self.reduce_big_files:
+            data = self._reduce_file(data, file_name)
+
         f = open(data_file_name, 'w')
         f.write(data)
         f.close()
 
         return data
 
+    def _reduce_file(self, data, file_name):
+        if mod_math.sqrt(len(data) / 2) == 3601:
+            mod_logging.info('Reducing file %s' % file_name)
+            reduced_data = mod_cstringio.StringIO()
+            values_written = 0
+            size = len(data)
+            for i in xrange(size / 2):
+                row = i / 3601
+                column = i % 3601
+                if row % 3 == 0 and column % 3 == 0:
+                    reduced_data.write(data[i * 2])
+                    reduced_data.write(data[i * 2 + 1])
+                    values_written += 1
+
+            result = reduced_data.getvalue()
+            assert mod_math.sqrt(len(result) / 2) == 1201
+            return result
+        return data
+
     def get_file_name(self, latitude, longitude):
         # Decide the file name:
         if latitude >= 0:
@@ -338,4 +363,3 @@ def parse_file_name_starting_position(self):
 	
     def __str__(self):
         return '[{0}:{1}]'.format(self.__class__, self.file_name)
-
