parser: fix 16 bit state limitation

The hfa stores next/check transitions in 16 bit fields to reduce memory
usage. However this means the state machine can on contain 2^16
states.

Allow the next/check tables to be 32 bit. This theoretically could allow
for 2^32 states however the base table uses the top 8 bits as flags
giving us only 2^24 bits to index into the next/check tables. With
most states having at least 1 transition this effectively caps the
number of states at 2^24.

To obtain 2^32 possible states a flags table needs to be added. Add
a skeleton around supporting a flags table, so we can note the remaining
work that needs to be done. This patch will only allow for 2^24 states.

Bug: https://gitlab.com/apparmor/apparmor/-/issues/419

Signed-off-by: John Johansen <john.johansen@canonical.com>

parser/common_optarg.c

@@ -71,6 +71,10 @@ optflag_table_t dumpflag_table[] = {
 	{ 1, "diff-progress", "Dump progress of differential encoding",
 	  DUMP_DFA_DIFF_PROGRESS | DUMP_DFA_DIFF_STATS },
 	{ 1, "rule-merge", "dump information about rule merging", DUMP_RULE_MERGE},
+	{ 1, "state32", "Dump encoding 32 bit states",
+	  DUMP_DFA_STATE32 },
+	{ 1, "flags_table", "Dump encoding flags table",
+	  DUMP_DFA_FLAGS_TABLE },
 	{ 0, NULL, NULL, 0 },
 };
 
@@ -78,7 +82,8 @@ optflag_table_t dfaoptflag_table[] = {
 	{ 2, "0", "no optimizations",
 	  CONTROL_DFA_TREE_NORMAL | CONTROL_DFA_TREE_SIMPLE |
 	  CONTROL_DFA_MINIMIZE | CONTROL_DFA_REMOVE_UNREACHABLE |
-	  CONTROL_DFA_DIFF_ENCODE
+	  CONTROL_DFA_DIFF_ENCODE | CONTROL_DFA_STATE32 |
+	  CONTROL_DFA_FLAGS_TABLE
 	},
 	{ 1, "equiv", "use equivalent classes", CONTROL_DFA_EQUIV },
 	{ 1, "expr-normalize", "expression tree normalization",
@@ -102,6 +107,10 @@ optflag_table_t dfaoptflag_table[] = {
 	{ 1, "diff-encode", "Differentially encode transitions",
 	  CONTROL_DFA_DIFF_ENCODE },
 	{ 1, "rule-merge", "turn on rule merging", CONTROL_RULE_MERGE},
+	{ 1, "state32", "use 32 bit state transitions",
+	  CONTROL_DFA_STATE32 },
+	{ 1, "flags-table", "use independent flags table",
+	  CONTROL_DFA_FLAGS_TABLE },
 	{ 0, NULL, NULL, 0 },
 };
 

parser/libapparmor_re/README

@@ -29,6 +29,10 @@ DFA16
 DFA32
   default/next/check - are 32 bit tables
 
+  DFA32 is limited to 2^24 states, due to the upper 8 bits being used
+  as flags in the base table, unless the flags table is defined. When
+  the flags table is defined, DFA32 can have a full 2^32 states.
+
 In both DFA16 and DFA32
    base and accept are 32 bit tables.
 
@@ -38,8 +42,9 @@ fields should be 0.
 State 1 is the default start state. Alternate start states are stored
 external to the state machine.
 
-The base table uses the lower 24 bits as index into the next/check tables,
+If the flags table is not defined, the base table uses the lower 24
-and the upper 8 bits are used as flags.
+bits as index into the next/check tables, and the upper 8 bits are used
+as flags.
 
 The currently defined flags are
 #define MATCH_FLAG_DIFF_ENCODE 0x80000000

parser/libapparmor_re/aare_rules.cc

@@ -344,7 +344,7 @@ void *aare_rules::create_dfablob(size_t *size, int *min_match_len,
 			*size = 0;
 			return NULL;
 		}
-		chfa->flex_table(stream);
+		chfa->flex_table(stream, opts);
 		delete (chfa);
 	}
 	catch(int error) {
@@ -417,7 +417,7 @@ void *aare_rules::create_welded_dfablob(aare_rules *file_rules,
 		policy_chfa->weld_file_to_policy(*file_chfa, *new_start,
 						 extended_perms, prompt,
 						 perms_table, file_perms);
-		policy_chfa->flex_table(stream);
+		policy_chfa->flex_table(stream, opts);
 	}
 	catch(int error) {
 		delete (file_chfa);

parser/libapparmor_re/apparmor_re.h

@@ -31,6 +31,8 @@
 #define CONTROL_DFA_TRANS_HIGH		(1 << 8)
 #define CONTROL_DFA_DIFF_ENCODE		(1 << 9)
 #define CONTROL_RULE_MERGE		(1 << 10)
+#define CONTROL_DFA_STATE32		(1 << 11)
+#define CONTROL_DFA_FLAGS_TABLE		(1 << 12)
 
 
 #define DUMP_DFA_DIFF_PROGRESS		(1 << 0)
@@ -56,5 +58,7 @@
 #define DUMP_DFA_RULE_EXPR 		(1 << 20)
 #define DUMP_DFA_NODE_TO_DFA 		(1 << 21)
 #define DUMP_RULE_MERGE			(1 << 22)
+#define DUMP_DFA_STATE32		(1 << 23)
+#define DUMP_DFA_FLAGS_TABLE		(1 << 24)
 
 #endif /* APPARMOR_RE_H */

parser/libapparmor_re/chfa.cc

@@ -396,7 +396,9 @@ template<class Iter>
 	os << fill64(sizeof(td) + sizeof(*pos) * size);
 }
 
-void CHFA::flex_table(ostream &os)
+template<class STATE_TYPE>
+void flex_table_serialize(CHFA &chfa, ostream &os,
+			  uint32_t max_size)
 {
 	const char th_version[] = "notflex";
 	struct table_set_header th = { 0, 0, 0, 0 };
@@ -405,16 +407,15 @@ void CHFA::flex_table(ostream &os)
 	 * Change the following two data types to adjust the maximum flex
 	 * table size.
 	 */
-	typedef uint16_t state_t;
 	typedef uint32_t trans_t;
 
-	if (default_base.size() >= (state_t) - 1) {
+	if (chfa.default_base.size() >= (max_size)) {
-		cerr << "Too many states (" << default_base.size() << ") for "
+		cerr << "Too many states (" << chfa.default_base.size() << ") for "
 		    "type state_t\n";
 		exit(1);
 	}
-	if (next_check.size() >= (trans_t) - 1) {
+	if (chfa.next_check.size() >= (trans_t) - 1) {
-		cerr << "Too many transitions (" << next_check.size()
+		cerr << "Too many transitions (" << chfa.next_check.size()
 		     << ") for " "type trans_t\n";
 		exit(1);
 	}
@@ -424,25 +425,25 @@ void CHFA::flex_table(ostream &os)
 	 * using the generic write_flex_table() routine.
 	 */
 	vector<uint8_t> equiv_vec;
-	if (eq.size()) {
+	if (chfa.eq.size()) {
 		equiv_vec.resize(256);
-		for (map<transchar, transchar>::iterator i = eq.begin(); i != eq.end(); i++) {
+		for (map<transchar, transchar>::iterator i = chfa.eq.begin(); i != chfa.eq.end(); i++) {
 			equiv_vec[i->first.c] = i->second.c;
 		}
 	}
 
-	vector<state_t> default_vec;
+	vector<STATE_TYPE> default_vec;
 	vector<trans_t> base_vec;
-	for (DefaultBase::iterator i = default_base.begin(); i != default_base.end(); i++) {
+	for (DefaultBase::iterator i = chfa.default_base.begin(); i != chfa.default_base.end(); i++) {
-		default_vec.push_back(num[i->first]);
+		default_vec.push_back(chfa.num[i->first]);
 		base_vec.push_back(i->second);
 	}
 
-	vector<state_t> next_vec;
+	vector<STATE_TYPE> next_vec;
-	vector<state_t> check_vec;
+	vector<STATE_TYPE> check_vec;
-	for (NextCheck::iterator i = next_check.begin(); i != next_check.end(); i++) {
+	for (NextCheck::iterator i = chfa.next_check.begin(); i != chfa.next_check.end(); i++) {
-		next_vec.push_back(num[i->first]);
+		next_vec.push_back(chfa.num[i->first]);
-		check_vec.push_back(num[i->second]);
+		check_vec.push_back(chfa.num[i->second]);
 	}
 
 	/* Write the actual flex parser table. */
@@ -450,25 +451,34 @@ void CHFA::flex_table(ostream &os)
 	// sizeof(th_version) includes trailing \0
 	size_t hsize = pad64(sizeof(th) + sizeof(th_version));
 	th.th_magic = htonl(YYTH_REGEX_MAGIC);
-	th.th_flags = htons(chfaflags);
+	th.th_flags = htons(chfa.chfaflags);
 	th.th_hsize = htonl(hsize);
 	th.th_ssize = htonl(hsize +
-			    flex_table_size(accept.begin(), accept.end()) +
+			    flex_table_size(chfa.accept.begin(),
-			    (accept2.size() ? flex_table_size(accept2.begin(), accept2.end()) : 0) +
+					    chfa.accept.end()) +
-			    (eq.size() ? flex_table_size(equiv_vec.begin(), equiv_vec.end()) : 0) +
+			    (chfa.accept2.size() ?
-			    flex_table_size(base_vec.begin(), base_vec.end()) +
+			     flex_table_size(chfa.accept2.begin(),
-			    flex_table_size(default_vec.begin(), default_vec.end()) +
+					     chfa.accept2.end()) : 0) +
+			    (chfa.eq.size() ?
+			     flex_table_size(equiv_vec.begin(),
+					     equiv_vec.end()) : 0) +
+			    flex_table_size(base_vec.begin(),
+					    base_vec.end()) +
+			    flex_table_size(default_vec.begin(),
+					    default_vec.end()) +
 			    flex_table_size(next_vec.begin(), next_vec.end()) +
-			    flex_table_size(check_vec.begin(), check_vec.end()));
+			    flex_table_size(check_vec.begin(),
+					    check_vec.end()));
 	os.write((char *)&th, sizeof(th));
 	os.write(th_version, sizeof(th_version));
 	os << fill64(sizeof(th) + sizeof(th_version));
 
-	write_flex_table(os, YYTD_ID_ACCEPT, accept.begin(), accept.end());
+	write_flex_table(os, YYTD_ID_ACCEPT, chfa.accept.begin(),
-	if (accept2.size())
+			 chfa.accept.end());
-		write_flex_table(os, YYTD_ID_ACCEPT2, accept2.begin(),
+	if (chfa.accept2.size())
-				 accept2.end());
+		write_flex_table(os, YYTD_ID_ACCEPT2, chfa.accept2.begin(),
-	if (eq.size())
+				 chfa.accept2.end());
+	if (chfa.eq.size())
 		write_flex_table(os, YYTD_ID_EC, equiv_vec.begin(),
 				 equiv_vec.end());
 	write_flex_table(os, YYTD_ID_BASE, base_vec.begin(), base_vec.end());
@@ -477,6 +487,29 @@ void CHFA::flex_table(ostream &os)
 	write_flex_table(os, YYTD_ID_CHK, check_vec.begin(), check_vec.end());
 }
 
+void CHFA::flex_table(ostream &os, optflags const &opts) {
+
+	if (opts.control & CONTROL_DFA_STATE32) {
+// TODO: implement support for flags in separate table
+//		if (opts.control & CONTROL_DFA_FLAGS_TABLE) {
+//			if (opts.dump & DUMP_FLAGS_TABLE)
+//				cerr << "using flags table\n";
+//			flex_table_serialize(os, uint32_t, (1 << 32) - 1);
+//		} else { /* only 24 bits available */
+		if (opts.dump & DUMP_DFA_STATE32)
+			cerr << "using 32 bit state tables, embedded flags\n";
+		flex_table_serialize<uint32_t>(*this, os, (1 << 24) - 1);
+	} else {
+		if (opts.control & CONTROL_DFA_FLAGS_TABLE) {
+			cerr << "Flags table specified when using 16 bit state\n";
+			exit(1);
+		}
+		if (opts.dump & DUMP_DFA_STATE32)
+			cerr << "using 16 bit state tables, embedded flags\n";
+		flex_table_serialize<uint16_t>(*this, os, (1 << 16) - 1);
+	}
+}
+
 /*
  * @file_chfa: chfa to add on to the policy chfa
  * @new_start: new start state for where the @file_dfa is in the new chfa

parser/libapparmor_re/chfa.h

@@ -34,15 +34,16 @@
 
 using namespace std;
 
+typedef vector<pair<const State *, size_t> > DefaultBase;
+typedef vector<pair<const State *, const State *> > NextCheck;
+
 class CHFA {
-	typedef vector<pair<const State *, size_t> > DefaultBase;
-	typedef vector<pair<const State *, const State *> > NextCheck;
       public:
 	CHFA(void);
 	CHFA(DFA &dfa, map<transchar, transchar> &eq, optflags const &opts,
 	     bool permindex, bool prompt);
 	void dump(ostream & os);
-	void flex_table(ostream &os);
+	void flex_table(ostream &os, optflags const &opts);
 	void init_free_list(vector<pair<size_t, size_t> > &free_list,
 			    size_t prev, size_t start);
 	bool fits_in(vector<pair<size_t, size_t> > &free_list, size_t base,
@@ -54,7 +55,9 @@ class CHFA {
 				 vector <aa_perms>  &policy_perms,
 				 vector <aa_perms> &file_perms);
 
-      private:
+	// private:
+	// sigh templates suck, friend declaration does not work so for now
+	// make these public
 	vector<uint32_t> accept;
 	vector<uint32_t> accept2;
 	DefaultBase default_base;
@@ -62,9 +65,10 @@ class CHFA {
 	const State *start;
 	map<const State *, size_t> num;
 	map<transchar, transchar> eq;
+	unsigned int chfaflags;
+      private:
 	transchar max_eq;
 	ssize_t first_free;
-	unsigned int chfaflags;
 };
 
 #endif /* __LIBAA_RE_CHFA_H */

parser/parser.h

@@ -364,6 +364,8 @@ extern int kernel_supports_promptdev;
 extern int kernel_supports_permstable32;
 extern int kernel_supports_permstable32_v1;
 extern int prompt_compat_mode;
+extern int kernel_supports_state32;
+extern int kernel_supports_flags_table;
 extern int conf_verbose;
 extern int conf_quiet;
 extern int names_only;

parser/parser_common.c

@@ -91,6 +91,8 @@ int kernel_supports_promptdev = 0;	/* prompt via audit perms */
 int kernel_supports_permstable32 = 0;	/* extended permissions */
 int kernel_supports_permstable32_v1 = 0;	/* extended permissions */
 int prompt_compat_mode = PROMPT_COMPAT_UNKNOWN;
+int kernel_supports_state32 = 0;	/* 32 bit state table entries */
+int kernel_supports_flags_table = 0;	/* state flags stored in table */
 int conf_verbose = 0;
 int conf_quiet = 0;
 int names_only = 0;

parser/parser_main.c

@@ -1564,6 +1564,10 @@ static bool get_kernel_features(struct aa_features **features)
 						       "policy/set_load");
 	kernel_supports_diff_encode = aa_features_supports(*features,
 							   "policy/diff_encode");
+	kernel_supports_state32 = aa_features_supports(*features,
+						       "policy/state32");
+	kernel_supports_flags_table = aa_features_supports(*features,
+						     "policy/flags_table");
 	kernel_supports_oob = aa_features_supports(*features,
 						   "policy/outofband");
 
@@ -1590,6 +1594,14 @@ static bool get_kernel_features(struct aa_features **features)
 		/* clear diff_encode because it is not supported */
 		parseopts.control &= ~CONTROL_DFA_DIFF_ENCODE;
 
+	if (!kernel_supports_state32)
+		parseopts.control &= ~CONTROL_DFA_STATE32;
+	if (!kernel_supports_flags_table || !kernel_supports_state32)
+		/* if only encoding 16 bit states, don't waste space on
+		 * a flags table
+		 */
+		parseopts.control &= ~CONTROL_DFA_FLAGS_TABLE;
+
 	return true;
 }