factor out function for getting top level directory of cloudinit (#1136)

Add a test helper to get top level directory

Many tests need to get the location of files & dirs within the cloud-init
project directory.

Tests implement this in various different ways, and often those ways
depend on the current working directory of the pytest invocation.
Create helper functions (and tests) that gets the path of the top
directory or any sub directory under the top directory. This function
does not depend on the environment.

10 files changed, 111 insertions, 30 deletions

diff --git a/tests/unittests/config/test_cc_chef.py b/tests/unittests/config/test_cc_chef.py
index 060293c8..1c90a4fc 100644
--- a/tests/unittests/config/test_cc_chef.py
+++ b/tests/unittests/config/test_cc_chef.py
@@ -9,13 +9,18 @@ from cloudinit.config import cc_chef
 from cloudinit import util
 
 from tests.unittests.helpers import (
-    HttprettyTestCase, FilesystemMockingTestCase, mock, skipIf)
+    HttprettyTestCase,
+    FilesystemMockingTestCase,
+    mock,
+    skipIf,
+    cloud_init_project_dir,
+)
 
 from tests.unittests.util import get_cloud
 
 LOG = logging.getLogger(__name__)
 
-CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"])
+CLIENT_TEMPL = cloud_init_project_dir("templates/chef_client.rb.tmpl")
 
 # This is adjusted to use http because using with https causes issue
 # in some openssl/httpretty combinations.
@@ -138,7 +143,7 @@ class TestChef(FilesystemMockingTestCase):
         Chef::Log::Formatter.show_time = true
         encrypted_data_bag_secret  "/etc/chef/encrypted_data_bag_secret"
         """
-        tpl_file = util.load_file('templates/chef_client.rb.tmpl')
+        tpl_file = util.load_file(CLIENT_TEMPL)
         self.patchUtils(self.tmp)
         self.patchOS(self.tmp)
 
@@ -200,7 +205,7 @@ class TestChef(FilesystemMockingTestCase):
     @skipIf(not os.path.isfile(CLIENT_TEMPL),
             CLIENT_TEMPL + " is not available")
     def test_template_deletes(self):
-        tpl_file = util.load_file('templates/chef_client.rb.tmpl')
+        tpl_file = util.load_file(CLIENT_TEMPL)
         self.patchUtils(self.tmp)
         self.patchOS(self.tmp)
 
@@ -222,7 +227,7 @@ class TestChef(FilesystemMockingTestCase):
             CLIENT_TEMPL + " is not available")
     def test_validation_cert_and_validation_key(self):
         # test validation_cert content is written to validation_key path
-        tpl_file = util.load_file('templates/chef_client.rb.tmpl')
+        tpl_file = util.load_file(CLIENT_TEMPL)
         self.patchUtils(self.tmp)
         self.patchOS(self.tmp)
 
@@ -245,7 +250,7 @@ class TestChef(FilesystemMockingTestCase):
 
     def test_validation_cert_with_system(self):
         # test validation_cert content is not written over system file
-        tpl_file = util.load_file('templates/chef_client.rb.tmpl')
+        tpl_file = util.load_file(CLIENT_TEMPL)
         self.patchUtils(self.tmp)
         self.patchOS(self.tmp)
 
diff --git a/tests/unittests/config/test_cc_resolv_conf.py b/tests/unittests/config/test_cc_resolv_conf.py
index 0aa90a23..ab2de17a 100644
--- a/tests/unittests/config/test_cc_resolv_conf.py
+++ b/tests/unittests/config/test_cc_resolv_conf.py
@@ -114,7 +114,7 @@ class TestResolvConf(t_help.FilesystemMockingTestCase):
 class TestGenerateResolvConf:
 
     dist = MockDistro()
-    tmpl_fn = "templates/resolv.conf.tmpl"
+    tmpl_fn = t_help.cloud_init_project_dir("templates/resolv.conf.tmpl")
 
     @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file")
     def test_dist_resolv_conf_fn(self, m_render_to_file):
diff --git a/tests/unittests/config/test_cc_update_etc_hosts.py b/tests/unittests/config/test_cc_update_etc_hosts.py
index 77a7f78f..35ad6413 100644
--- a/tests/unittests/config/test_cc_update_etc_hosts.py
+++ b/tests/unittests/config/test_cc_update_etc_hosts.py
@@ -55,7 +55,10 @@ class TestHostsFile(t_help.FilesystemMockingTestCase):
             'manage_etc_hosts': 'template',
             'hostname': 'cloud-init.test.us'
         }
-        shutil.copytree('templates', '%s/etc/cloud/templates' % self.tmp)
+        shutil.copytree(
+            t_help.cloud_init_project_dir('templates'),
+            '%s/etc/cloud/templates' % self.tmp,
+        )
         distro = self._fetch_distro('sles')
         paths = helpers.Paths({})
         paths.template_tpl = '%s' % self.tmp + '/etc/cloud/templates/%s.tmpl'
diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py
index f90e0f62..ed7ab527 100644
--- a/tests/unittests/config/test_schema.py
+++ b/tests/unittests/config/test_schema.py
@@ -12,7 +12,6 @@ from pathlib import Path
 from textwrap import dedent
 from yaml import safe_load
 
-import cloudinit
 from cloudinit.config.schema import (
     CLOUD_CONFIG_HEADER,
     SchemaValidationError,
@@ -27,7 +26,12 @@ from cloudinit.config.schema import (
     MetaSchema,
 )
 from cloudinit.util import write_file
-from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema
+from tests.unittests.helpers import (
+    CiTestCase,
+    mock,
+    skipUnlessJsonSchema,
+    cloud_init_project_dir,
+)
 
 
 def get_schemas() -> dict:
@@ -50,7 +54,10 @@ def get_module_variable(var_name) -> dict:
     """Inspect modules and get variable from module matching var_name"""
     schemas = {}
 
-    files = list(Path("../../cloudinit/config/").glob("cc_*.py"))
+    files = list(
+        Path(cloud_init_project_dir("../../cloudinit/config/")).glob("cc_*.py")
+    )
+
     modules = [mod.stem for mod in files]
 
     for module in modules:
@@ -616,8 +623,7 @@ class TestMain:
 
 
 def _get_meta_doc_examples():
-    examples_dir = Path(
-        cloudinit.__file__).parent.parent / 'doc' / 'examples'
+    examples_dir = Path(cloud_init_project_dir('doc/examples'))
     assert examples_dir.is_dir()
 
     all_text_files = (f for f in examples_dir.glob('cloud-config*.txt')
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index ccd56793..e9afbd36 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -12,10 +12,12 @@ import sys
 import tempfile
 import time
 import unittest
+from pathlib import Path
 from contextlib import ExitStack, contextmanager
 from unittest import mock
 from unittest.util import strclass
 
+import cloudinit
 from cloudinit.config.schema import (
     SchemaValidationError, validate_cloudconfig_schema)
 from cloudinit import cloud
@@ -462,7 +464,7 @@ def wrap_and_call(prefix, mocks, func, *args, **kwargs):
 
 
 def resourceLocation(subname=None):
-    path = os.path.join('tests', 'data')
+    path = cloud_init_project_dir('tests/data')
     if not subname:
         return path
     return os.path.join(path, subname)
@@ -504,4 +506,22 @@ if not hasattr(mock.Mock, 'assert_not_called'):
             raise AssertionError(msg)
     mock.Mock.assert_not_called = __mock_assert_not_called
 
+
+def get_top_level_dir() -> Path:
+    """Return the absolute path to the top cloudinit project directory
+
+    @return Path('<top-cloudinit-dir>')
+    """
+    return Path(cloudinit.__file__).parent.parent.resolve()
+
+
+def cloud_init_project_dir(sub_path: str) -> str:
+    """Get a path within the cloudinit project directory
+
+    @return str of the combined path
+
+    Example: cloud_init_project_dir("my/path") -> "/path/to/cloud-init/my/path"
+    """
+    return str(get_top_level_dir() / sub_path)
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/sources/vmware/test_vmware_config_file.py b/tests/unittests/sources/vmware/test_vmware_config_file.py
index 54de113e..1d66ab4a 100644
--- a/tests/unittests/sources/vmware/test_vmware_config_file.py
+++ b/tests/unittests/sources/vmware/test_vmware_config_file.py
@@ -16,15 +16,21 @@ from cloudinit.sources.DataSourceOVF import get_network_config_from_conf
 from cloudinit.sources.DataSourceOVF import read_vmware_imc
 from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum
 from cloudinit.sources.helpers.vmware.imc.config import Config
-from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile
+from cloudinit.sources.helpers.vmware.imc.config_file import (
+    ConfigFile as WrappedConfigFile,
+)
 from cloudinit.sources.helpers.vmware.imc.config_nic import gen_subnet
 from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator
-from tests.unittests.helpers import CiTestCase
+from tests.unittests.helpers import CiTestCase, cloud_init_project_dir
 
 logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
 logger = logging.getLogger(__name__)
 
 
+def ConfigFile(path: str):
+    return WrappedConfigFile(cloud_init_project_dir(path))
+
+
 class TestVmwareConfigFile(CiTestCase):
 
     def test_utility_methods(self):
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 62c3e403..eb8992d9 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -9,8 +9,12 @@ from cloudinit import safeyaml
 from cloudinit import subp
 from cloudinit import util
 from tests.unittests.helpers import (
-    CiTestCase, dir2dict, populate_dir, populate_dir_with_ts)
-
+    CiTestCase,
+    dir2dict,
+    populate_dir,
+    populate_dir_with_ts,
+    cloud_init_project_dir,
+)
 from cloudinit.sources import DataSourceIBMCloud as ds_ibm
 from cloudinit.sources import DataSourceSmartOS as ds_smartos
 from cloudinit.sources import DataSourceOracle as ds_oracle
@@ -92,7 +96,7 @@ CallReturn = namedtuple('CallReturn',
 
 
 class DsIdentifyBase(CiTestCase):
-    dsid_path = os.path.realpath('tools/ds-identify')
+    dsid_path = cloud_init_project_dir('tools/ds-identify')
     allowed_subp = ['sh']
 
     def call(self, rootd=None, mocks=None, func="main", args=None, files=None,
diff --git a/tests/unittests/test_helpers.py b/tests/unittests/test_helpers.py
index c6f9b94a..f491f8cd 100644
--- a/tests/unittests/test_helpers.py
+++ b/tests/unittests/test_helpers.py
@@ -3,6 +3,7 @@
 """Tests of the built-in user data handlers."""
 
 import os
+from pathlib import Path
 
 from tests.unittests import helpers as test_helpers
 
@@ -34,4 +35,36 @@ class TestPaths(test_helpers.ResourceUsingTestCase):
 
         self.assertIsNone(mypaths.get_ipath())
 
+
+class Testcloud_init_project_dir:
+    top_dir = test_helpers.get_top_level_dir()
+
+    @staticmethod
+    def _get_top_level_dir_alt_implementation():
+        """Alternative implementation for comparing against.
+
+        Note: Recursively searching for .git/ fails during build tests due to
+        .git not existing. This implementation assumes that ../../../ is the
+        relative path to the cloud-init project directory form this file.
+        """
+        out = Path(__file__).parent.parent.parent.resolve()
+        return out
+
+    def test_top_level_dir(self):
+        """Assert the location of the top project directory is correct"""
+        assert (self.top_dir ==
+                self._get_top_level_dir_alt_implementation())
+
+    def test_cloud_init_project_dir(self):
+        """Assert cloud_init_project_dir produces an expected location
+
+        Compare the returned value to an alternate (naive) implementation
+        """
+        assert (
+            str(Path(self.top_dir, "test"))
+            == test_helpers.cloud_init_project_dir("test")
+            == str(Path(self._get_top_level_dir_alt_implementation(), "test"))
+        )
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_render_cloudcfg.py b/tests/unittests/test_render_cloudcfg.py
index 00d50e66..b2222747 100644
--- a/tests/unittests/test_render_cloudcfg.py
+++ b/tests/unittests/test_render_cloudcfg.py
@@ -1,12 +1,12 @@
 """Tests for tools/render-cloudcfg"""
 
-import os
 import sys
 
 import pytest
 
 from cloudinit import subp
 from cloudinit import util
+from tests.unittests.helpers import cloud_init_project_dir
 
 # TODO(Look to align with tools.render-cloudcfg or cloudinit.distos.OSFAMILIES)
 DISTRO_VARIANTS = ["amazon", "arch", "centos", "debian", "eurolinux", "fedora",
@@ -17,8 +17,8 @@ DISTRO_VARIANTS = ["amazon", "arch", "centos", "debian", "eurolinux", "fedora",
 @pytest.mark.allow_subp_for(sys.executable)
 class TestRenderCloudCfg:
 
-    cmd = [sys.executable, os.path.realpath('tools/render-cloudcfg')]
-    tmpl_path = os.path.realpath('config/cloud.cfg.tmpl')
+    cmd = [sys.executable, cloud_init_project_dir('tools/render-cloudcfg')]
+    tmpl_path = cloud_init_project_dir('config/cloud.cfg.tmpl')
 
     @pytest.mark.parametrize('variant', (DISTRO_VARIANTS))
     def test_variant_sets_distro_in_cloud_cfg(self, variant, tmpdir):
diff --git a/tests/unittests/test_subp.py b/tests/unittests/test_subp.py
index ec513d01..572510d7 100644
--- a/tests/unittests/test_subp.py
+++ b/tests/unittests/test_subp.py
@@ -10,7 +10,7 @@ import stat
 from unittest import mock
 
 from cloudinit import subp, util
-from tests.unittests.helpers import CiTestCase
+from tests.unittests.helpers import CiTestCase, get_top_level_dir
 
 
 BASH = subp.which('bash')
@@ -232,13 +232,17 @@ class TestSubp(CiTestCase):
         the default encoding will be set to ascii.  In such an environment
         Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError.
         """
-        python_prog = '\n'.join([
-            'import json, sys',
-            'from cloudinit.subp import subp',
-            'data = sys.stdin.read()',
-            'cmd = json.loads(data)',
-            'subp(cmd, capture=False)',
-            ''])
+        python_prog = '\n'.join(
+            [
+                'import json, sys',
+                'sys.path.insert(0, "{}")'.format(get_top_level_dir()),
+                'from cloudinit.subp import subp',
+                'data = sys.stdin.read()',
+                'cmd = json.loads(data)',
+                'subp(cmd, capture=False)',
+                '',
+            ]
+        )
         cmd = [BASH, '-c', 'echo -n "$@"', '--',
                self.utf8_valid.decode("utf-8")]
         python_subp = [sys.executable, '-c', python_prog]