Let aa-cleanprof remove duplicate preamble rules

Technically, this is done in the new function
delete_preamble_duplicates() in ProfileList.

Also add some tests to ensure this works as expected.

utils/apparmor/cleanprofile.py

@@ -46,9 +46,15 @@ class CleanProf(object):
 
     def remove_duplicate_rules(self, program):
         #Process the profile of the program
+
+        deleted = 0
+
+        # remove duplicate rules from the preamble
+        deleted += self.profile.active_profiles.delete_preamble_duplicates(self.profile.filename)
+
         #Process every hat in the profile individually
         file_includes = list(self.profile.filelist[self.profile.filename]['include'].keys())
-        deleted = 0
+
         for hat in sorted(self.profile.aa[program].keys()):
             #The combined list of includes from profile and the file
             includes = list(self.profile.aa[program][hat]['include'].keys()) + file_includes

utils/apparmor/profile_list.py

@@ -116,6 +116,19 @@ class ProfileList:
 
         self.files[filename]['inc_ie'].add(inc_rule)
 
+    def delete_preamble_duplicates(self, filename):
+        ''' Delete duplicates in the preamble of the given profile file '''
+
+        if not self.files.get(filename):
+            raise AppArmorBug('%s not listed in ProfileList files' % filename)
+
+        deleted = 0
+
+        for r_type in ['abi', 'inc_ie']:  # TODO: don't hardcode
+            deleted += self.files[filename][r_type].delete_duplicates(None)  # None means not to check includes -- TODO check if this makes sense for all preamble rule types
+
+        return deleted
+
     def get_raw(self, filename, depth=0):
         ''' Get the preamble for the given profile filename (in original formatting) '''
         if not self.files.get(filename):

utils/test/cleanprof_test.in

@@ -3,6 +3,8 @@
 
 #include  if  exists <tunables/nothing>
 
+  #include if exists <tunables/global>
+  include if exists <tunables/global>
 
 		alias /foo    ->    /bar	,
 

utils/test/test-profile-list.py

@@ -149,6 +149,21 @@ class TestAdd_inc_ie(AATest):
             self.pl.add_inc_ie('/etc/apparmor.d/bin.foo', 'tunables/global')  # str insteadd of IncludeRule
         self.assertEqual(list(self.pl.files.keys()), [])
 
+    def test_dedup_inc_ie_1(self):
+        self.pl.add_inc_ie('/etc/apparmor.d/bin.foo', IncludeRule.parse('include <tunables/global>'))
+        self.pl.add_inc_ie('/etc/apparmor.d/bin.foo', IncludeRule.parse('#include if exists <tunables/global>  # comment'))
+        self.pl.add_inc_ie('/etc/apparmor.d/bin.foo', IncludeRule.parse('   #include         <tunables/global>    '))
+        deleted = self.pl.delete_preamble_duplicates('/etc/apparmor.d/bin.foo')
+        self.assertEqual(deleted, 2)
+        self.assertEqual(list(self.pl.files.keys()), ['/etc/apparmor.d/bin.foo'])
+        self.assertEqual(self.pl.get_clean('/etc/apparmor.d/bin.foo'), ['include <tunables/global>', ''])
+        self.assertEqual(self.pl.get_raw('/etc/apparmor.d/bin.foo'), ['include <tunables/global>', ''])
+
+    def test_dedup_error_1(self):
+        with self.assertRaises(AppArmorBug):
+            self.pl.delete_preamble_duplicates('/file/not/found')
+        self.assertEqual(list(self.pl.files.keys()), [])
+
 class TestAdd_abi(AATest):
     def AASetup(self):
         self.pl = ProfileList()
@@ -173,6 +188,15 @@ class TestAdd_abi(AATest):
             self.pl.add_abi('/etc/apparmor.d/bin.foo', 'abi/4.19')  # str insteadd of AbiRule
         self.assertEqual(list(self.pl.files.keys()), [])
 
+    def test_dedup_abi_1(self):
+        self.pl.add_abi('/etc/apparmor.d/bin.foo', AbiRule.parse('abi <abi/4.19>,'))
+        self.pl.add_abi('/etc/apparmor.d/bin.foo', AbiRule.parse('   abi     <abi/4.19>  ,  # comment'))
+        self.assertEqual(list(self.pl.files.keys()), ['/etc/apparmor.d/bin.foo'])
+        deleted = self.pl.delete_preamble_duplicates('/etc/apparmor.d/bin.foo')
+        self.assertEqual(deleted, 1)
+        self.assertEqual(self.pl.get_clean_first('/etc/apparmor.d/bin.foo'), ['abi <abi/4.19>,', ''])  # TODO switch to get_clean() once merged
+        self.assertEqual(self.pl.get_raw('/etc/apparmor.d/bin.foo'), ['abi <abi/4.19>,', ''])
+
 class TestAdd_alias(AATest):
     def AASetup(self):
         self.pl = ProfileList()
@@ -210,6 +234,9 @@ class TestAdd_alias(AATest):
             self.pl.add_alias('/etc/apparmor.d/bin.foo', '/foo', None)  # target None insteadd of str
         self.assertEqual(list(self.pl.files.keys()), [])
 
+#    def test_dedup_alias_1(self):
+#        TODO: implement after fixing alias handling (when a profile has two aliases with the same path on the left side)
+
 class TestGet(AATest):
     def AASetup(self):
         self.pl = ProfileList()