Also cleanup the space for the rbt nodes

As we are in overmem state we want to free more memory than we are
adding so we need to add in an allowance for the rbtnodes that may
have been added and the names stored with them.  There is the node
for the owner name and a possible ENT node if there was a node split.

(cherry picked from commit 7e462c2b26)

lib/dns/include/dns/rbt.h

@@ -992,4 +992,11 @@ dns__rbtnode_namelen(dns_rbtnode_t *node);
  * Returns the length of the full name of the node. Used only internally
  * and in unit tests.
  */
+
+unsigned int
+dns__rbtnode_getsize(dns_rbtnode_t *node);
+/*
+ * Return allocated size for a node.
+ */
+
 ISC_LANG_ENDDECLS

lib/dns/rbt.c

@@ -302,6 +302,13 @@ dns__rbtnode_namelen(dns_rbtnode_t *node) {
 	return (len);
 }
 
+unsigned int
+dns__rbtnode_getsize(dns_rbtnode_t *node) {
+	REQUIRE(DNS_RBTNODE_VALID(node));
+
+	return (NODE_SIZE(node));
+}
+
 /*
  * Initialize a red/black tree of trees.
  */

lib/dns/rbtdb.c

@@ -10201,7 +10201,9 @@ overmem_purge(dns_rbtdb_t *rbtdb, rdatasetheader_t *newheader,
 	      bool tree_locked) {
 	uint32_t locknum_start = rbtdb->lru_sweep++ % rbtdb->node_lock_count;
 	uint32_t locknum = locknum_start;
-	size_t purgesize = rdataset_size(newheader);
+	/* Size of added data, possible node and possible ENT node. */
+	size_t purgesize = rdataset_size(newheader) +
+			   2 * dns__rbtnode_getsize(newheader->node);
 	size_t purged = 0;
 	isc_stdtime_t min_last_used = 0;
 	size_t max_passes = 8;