Fix several ResourceWarning: unclosed file (#15746)

Summary:
Hello,

This is a patch to fix `ResourceWarning: unclosed file`.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/15746

Differential Revision: D13587286

Pulled By: soumith

fbshipit-source-id: 08ac34c5b51d9334867f65a2927bff11511553f3

diff --git a/.jenkins/pytorch/win-build.sh b/.jenkins/pytorch/win-build.sh
index b225eb9..9ca9fde 100755 (executable)
--- a/.jenkins/pytorch/win-build.sh
+++ b/.jenkins/pytorch/win-build.sh
@@ -29,8 +29,8 @@ IMAGE_COMMIT_TAG = os.getenv('IMAGE_COMMIT_TAG')
 
 session = boto3.session.Session()
 s3 = session.resource('s3')
-data = open(sys.argv[1], 'rb')
-s3.Bucket('ossci-windows-build').put_object(Key='pytorch/'+IMAGE_COMMIT_TAG+'.7z', Body=data)
+with open(sys.argv[1], 'rb') as data:
+  s3.Bucket('ossci-windows-build').put_object(Key='pytorch/'+IMAGE_COMMIT_TAG+'.7z', Body=data)
 object_acl = s3.ObjectAcl('ossci-windows-build','pytorch/'+IMAGE_COMMIT_TAG+'.7z')
 response = object_acl.put(ACL='public-read')
 

diff --git a/aten/src/ATen/gen.py b/aten/src/ATen/gen.py
index 0a627ea..3615700 100644 (file)
--- a/aten/src/ATen/gen.py
+++ b/aten/src/ATen/gen.py
@@ -428,8 +428,10 @@ def cmpfiles_with_eol_normalization(a, b, names):
     results = ([], [], [])    # match, mismatch, error
     for x in names:
         try:
-            ax = open(os.path.join(a, x), 'r').read().replace('\r\n', '\n').replace('\r', '\n')
-            bx = open(os.path.join(b, x), 'r').read().replace('\r\n', '\n').replace('\r', '\n')
+            with open(os.path.join(a, x)) as f:
+                ax = f.read().replace('\r\n', '\n').replace('\r', '\n')
+            with open(os.path.join(b, x)) as f:
+                bx = f.read().replace('\r\n', '\n').replace('\r', '\n')
             if ax == bx:
                 results[0].append(x)
             else:

diff --git a/caffe2/contrib/cuda-convnet2/make-data/make-data.py b/caffe2/contrib/cuda-convnet2/make-data/make-data.py
index 1861ceb..69d8f57 100644 (file)
--- a/caffe2/contrib/cuda-convnet2/make-data/make-data.py
+++ b/caffe2/contrib/cuda-convnet2/make-data/make-data.py
@@ -52,10 +52,8 @@ def pickle(filename, data):
         cPickle.dump(data, fo, protocol=cPickle.HIGHEST_PROTOCOL)
 
 def unpickle(filename):
-    fo = open(filename, 'r')
-    contents = cPickle.load(fo)
-    fo.close()
-    return contents
+    with open(filename) as fo:
+        return cPickle.load(fo)
 
 def partition_list(l, partition_size):
     divup = lambda a,b: (a + b - 1) / b

diff --git a/caffe2/contrib/cuda-convnet2/python_util/util.py b/caffe2/contrib/cuda-convnet2/python_util/util.py
index c0a8a4a..e88c044 100644 (file)
--- a/caffe2/contrib/cuda-convnet2/python_util/util.py
+++ b/caffe2/contrib/cuda-convnet2/python_util/util.py
@@ -28,34 +28,27 @@ def pickle(filename, data):
     if type(filename) == str:
         fo = open(filename, "w")
     
-    cPickle.dump(data, fo, protocol=cPickle.HIGHEST_PROTOCOL)
-    fo.close()
+    with fo:
+        cPickle.dump(data, fo, protocol=cPickle.HIGHEST_PROTOCOL)
     
 def unpickle(filename):
     if not os.path.exists(filename):
         raise UnpickleError("Path '%s' does not exist." % filename)
 
-    fo = open(filename, 'r')
-    z = StringIO()
-    file_size = os.fstat(fo.fileno()).st_size
-    # Read 1GB at a time to avoid overflow
-    while fo.tell() < file_size:
-        z.write(fo.read(1 << 30))
-    fo.close()
-    dict = cPickle.loads(z.getvalue())
-    z.close()
-    
-    return dict
+    with open(filename) as fo, StringIO() as z:
+        file_size = os.fstat(fo.fileno()).st_size
+        # Read 1GB at a time to avoid overflow
+        while fo.tell() < file_size:
+            z.write(fo.read(1 << 30))
+        return cPickle.loads(z.getvalue())
 
 def is_intel_machine():
     VENDOR_ID_REGEX = re.compile(r'^vendor_id\s+: (\S+)')
-    f = open('/proc/cpuinfo')
-    for line in f:
-        m = VENDOR_ID_REGEX.match(line)
-        if m:
-            f.close()
-            return m.group(1) == 'GenuineIntel'
-    f.close()
+    with open('/proc/cpuinfo') as f:
+        for line in f:
+            m = VENDOR_ID_REGEX.match(line)
+            if m:
+                return m.group(1) == 'GenuineIntel'
     return False
 
 # Returns the CPUs associated with a given GPU
@@ -69,9 +62,8 @@ def get_cpus_for_gpu(gpu):
             if line.startswith('Bus Location'):
                 bus_id = line.split(':', 1)[1].strip()
                 bus_id = bus_id[:7] + ':' + bus_id[8:]
-                ff = open('/sys/module/nvidia/drivers/pci:nvidia/%s/local_cpulist' % bus_id)
-                cpus_str = ff.readline()
-                ff.close()
+                with open('/sys/module/nvidia/drivers/pci:nvidia/%s/local_cpulist' % bus_id) as ff:
+                    cpus_str = ff.readline()
                 cpus = [cpu for s in cpus_str.split(',') for cpu in range(int(s.split('-')[0]),int(s.split('-')[1])+1)]
                 return cpus
     return [-1]

diff --git a/caffe2/perfkernels/hp_emblookup_codegen.py b/caffe2/perfkernels/hp_emblookup_codegen.py
index c1cbd4f..20f759c 100644 (file)
--- a/caffe2/perfkernels/hp_emblookup_codegen.py
+++ b/caffe2/perfkernels/hp_emblookup_codegen.py
@@ -311,7 +311,6 @@ elif opts.fused:
     filename = "embedding_lookup_fused_8bit_rowwise_avx2.cc"
 else:
     filename = "embedding_lookup_avx2.cc"
-fout = open(filename, "w")
 
 options = [
     ["int32_t", "int32_t", "float", "float", "float", "float"],
@@ -422,10 +421,10 @@ for o in options:
 
 code.append("} // namespace caffe2")
 
-for c in code:
-    # print(c, file = fout)
-    fout.write(c + "\n")
-fout.close()
+with open(filename, "w") as fout:
+    for c in code:
+        # print(c, file = fout)
+        fout.write(c + "\n")
 
 
 print("Created " + filename)

diff --git a/caffe2/python/caffe_translator.py b/caffe2/python/caffe_translator.py
index 0592ca0..63392c7 100644 (file)
--- a/caffe2/python/caffe_translator.py
+++ b/caffe2/python/caffe_translator.py
@@ -911,12 +911,10 @@ if __name__ == '__main__':
     output_init_net = args.init_net
     output_predict_net = args.predict_net
 
-    text_format.Merge(
-        open(input_proto, 'r').read(), caffenet
-    )
-    caffenet_pretrained.ParseFromString(
-        open(input_caffemodel, 'rb').read()
-    )
+    with open(input_proto) as f:
+        text_format.Merge(f.read(), caffenet)
+    with open(input_caffemodel, 'rb') as f:
+        caffenet_pretrained.ParseFromString(f.read())
     net, pretrained_params = TranslateModel(
         caffenet, caffenet_pretrained, is_test=True,
         remove_legacy_pad=args.remove_legacy_pad,

diff --git a/caffe2/python/caffe_translator_test.py b/caffe2/python/caffe_translator_test.py
index 70b3956..ff24afe 100644 (file)
--- a/caffe2/python/caffe_translator_test.py
+++ b/caffe2/python/caffe_translator_test.py
@@ -31,14 +31,11 @@ def setUpModule():
     # We will do all the computation stuff in the global space.
     caffenet = caffe_pb2.NetParameter()
     caffenet_pretrained = caffe_pb2.NetParameter()
-    text_format.Merge(
-        open('data/testdata/caffe_translator/deploy.prototxt').read(), caffenet
-    )
-    caffenet_pretrained.ParseFromString(
-        open(
-            'data/testdata/caffe_translator/bvlc_reference_caffenet.caffemodel')
-        .read()
-    )
+    with open('data/testdata/caffe_translator/deploy.prototxt') as f:
+        text_format.Merge(f.read(), caffenet)
+    with open('data/testdata/caffe_translator/'
+              'bvlc_reference_caffenet.caffemodel') as f:
+        caffenet_pretrained.ParseFromString(f.read())
     for remove_legacy_pad in [True, False]:
         net, pretrained_params = caffe_translator.TranslateModel(
             caffenet, caffenet_pretrained, is_test=True,

diff --git a/caffe2/python/utils.py b/caffe2/python/utils.py
index 0d1f9f8..d5c6355 100644 (file)
--- a/caffe2/python/utils.py
+++ b/caffe2/python/utils.py
@@ -231,7 +231,8 @@ def GetContentFromProtoString(s, function_map):
 
 def ConvertProtoToBinary(proto_class, filename, out_filename):
     """Convert a text file of the given protobuf class to binary."""
-    proto = TryReadProtoWithClass(proto_class, open(filename).read())
+    with open(filename) as f:
+        proto = TryReadProtoWithClass(proto_class, f.read())
     with open(out_filename, 'w') as fid:
         fid.write(proto.SerializeToString())
 

diff --git a/test/run_test.py b/test/run_test.py
index ff44060..cd11273 100644 (file)
--- a/test/run_test.py
+++ b/test/run_test.py
@@ -233,10 +233,10 @@ def test_distributed(executable, test_module, test_directory, options):
                 os.mkdir(os.path.join(tmp_dir, 'test_dir'))
                 if backend == 'mpi':
                     # test mpiexec for --noprefix option
-                    devnull = open(os.devnull, 'w')
-                    noprefix_opt = '--noprefix' if subprocess.call(
-                        'mpiexec -n 1 --noprefix bash -c ""', shell=True,
-                        stdout=devnull, stderr=subprocess.STDOUT) == 0 else ''
+                    with open(os.devnull, 'w') as devnull:
+                        noprefix_opt = '--noprefix' if subprocess.call(
+                            'mpiexec -n 1 --noprefix bash -c ""', shell=True,
+                            stdout=devnull, stderr=subprocess.STDOUT) == 0 else ''
 
                     mpiexec = ['mpiexec', '-n', '3', noprefix_opt] + executable