ComputerSurge/bind9
3
0
Fork 0
Code Pull Requests 1 Activity

Refactor zone dumping code to use netmgr async threadpools

Browse Source 
Previously, dumping the zones to the files were quantized, so it doesn't
slow down network IO processing.  With the introduction of network
manager asynchronous threadpools, we can move the IO intensive work to
use that API and we don't have to quantize the work anymore as it the
file IO won't block anything except other zone dumping processes.
This commit is contained in:
Ondřej Surý
2021-05-31 14:52:05 +02:00
committed by Ondřej Surý
parent 7670f98377
commit 8a5c62de83
5 changed files with 78 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/named/server.c
+2 -2
View File
@@ -11487,7 +11487,7 @@ resume:
";\n; Cache dump of view '%s' (cache %s)\n;\n",
dctx->view->view->name,
dns_cache_getname(dctx->view->view->cache));
result = dns_master_dumptostreaminc(
result = dns_master_dumptostreamasync(
dctx->mctx, dctx->cache, NULL, style, dctx->fp,
dctx->task, dumpdone, dctx, &dctx->mdctx);
if (result == DNS_R_CONTINUE) {
@@ -11547,7 +11547,7 @@ resume:
goto nextzone;
}
dns_db_currentversion(dctx->db, &dctx->version);
result = dns_master_dumptostreaminc(
result = dns_master_dumptostreamasync(
dctx->mctx, dctx->db, dctx->version, style,
dctx->fp, dctx->task, dumpdone, dctx,
&dctx->mdctx);
lib/dns/include/dns/masterdump.h
+10 -22
View File
@@ -243,11 +243,11 @@ dns_dumpctx_db(dns_dumpctx_t *dctx);
/*@{*/
isc_result_t
dns_master_dumptostreaminc(isc_mem_t *mctx, dns_db_t *db,
dns_dbversion_t * version,
const dns_master_style_t *style, FILE *f,
isc_task_t *task, dns_dumpdonefunc_t done,
void *done_arg, dns_dumpctx_t **dctxp);
dns_master_dumptostreamasync(isc_mem_t *mctx, dns_db_t *db,
dns_dbversion_t * version,
const dns_master_style_t *style, FILE *f,
isc_task_t *task, dns_dumpdonefunc_t done,
void *done_arg, dns_dumpctx_t **dctxp);
isc_result_t
dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
@@ -259,11 +259,6 @@ dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
* 'format'. If the format is dns_masterformat_text (the RFC1035 format),
* 'style' specifies the file style (e.g., &dns_master_style_default).
*
* dns_master_dumptostream() is an old form of dns_master_dumptostream3(),
* which always specifies the dns_masterformat_text format.
* dns_master_dumptostream2() is an old form which always specifies
* a NULL header.
*
* If 'format' is dns_masterformat_raw, then 'header' can contain
* information to be written to the file header.
*
@@ -276,7 +271,6 @@ dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
*
* Returns:
*\li ISC_R_SUCCESS
*\li ISC_R_CONTINUE dns_master_dumptostreaminc() only.
*\li ISC_R_NOMEMORY
*\li Any database or rrset iterator error.
*\li Any dns_rdata_totext() error code.
@@ -286,11 +280,11 @@ dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
/*@{*/
isc_result_t
dns_master_dumpinc(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
const dns_master_style_t *style, const char *filename,
isc_task_t *task, dns_dumpdonefunc_t done, void *done_arg,
dns_dumpctx_t **dctxp, dns_masterformat_t format,
dns_masterrawheader_t *header);
dns_master_dumpasync(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
const dns_master_style_t *style, const char *filename,
isc_task_t *task, dns_dumpdonefunc_t done, void *done_arg,
dns_dumpctx_t **dctxp, dns_masterformat_t format,
dns_masterrawheader_t *header);
isc_result_t
dns_master_dump(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
@@ -302,11 +296,6 @@ dns_master_dump(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
* 'format'. If the format is dns_masterformat_text (the RFC1035 format),
* 'style' specifies the file style (e.g., &dns_master_style_default).
*
* dns_master_dumpinc() and dns_master_dump() are old forms of _dumpinc3()
* and _dump3(), respectively, which always specify the dns_masterformat_text
* format. dns_master_dumpinc2() and dns_master_dump2() are old forms which
* always specify a NULL header.
*
* If 'format' is dns_masterformat_raw, then 'header' can contain
* information to be written to the file header.
*
@@ -314,7 +303,6 @@ dns_master_dump(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
*
* Returns:
*\li ISC_R_SUCCESS
*\li ISC_R_CONTINUE dns_master_dumpinc() only.
*\li ISC_R_NOMEMORY
*\li Any database or rrset iterator error.
*\li Any dns_rdata_totext() error code.
lib/dns/masterdump.c
+63 -83
View File
@@ -265,8 +265,8 @@ struct dns_dumpctx {
isc_task_t *task;
dns_dumpdonefunc_t done;
void *done_arg;
unsigned int nodes;
/* dns_master_dumpinc() */
/* dns_master_dumpasync() */
isc_result_t result;
char *file;
char *tmpfile;
dns_masterformat_t format;
@@ -1343,7 +1343,7 @@ dump_rdatasets_map(isc_mem_t *mctx, const dns_name_t *name,
static const int initial_buffer_length = 1200;
static isc_result_t
dumptostreaminc(dns_dumpctx_t *dctx);
dumptostream(dns_dumpctx_t *dctx);
static void
dumpctx_destroy(dns_dumpctx_t *dctx) {
@@ -1486,27 +1486,23 @@ closeandrename(FILE *f, isc_result_t result, const char *temp,
return (result);
}
/*
* This will run in a libuv threadpool thread.
*/
static void
dump_quantum(isc_task_t *task, isc_event_t *event) {
isc_result_t result;
isc_result_t tresult;
dns_dumpctx_t *dctx;
REQUIRE(event != NULL);
dctx = event->ev_arg;
master_dump_cb(void *data) {
isc_result_t result = ISC_R_UNSET;
dns_dumpctx_t *dctx = data;
REQUIRE(DNS_DCTX_VALID(dctx));
if (atomic_load_acquire(&dctx->canceled)) {
result = ISC_R_CANCELED;
} else {
result = dumptostreaminc(dctx);
}
if (result == DNS_R_CONTINUE) {
event->ev_arg = dctx;
isc_task_send(task, &event);
return;
result = dumptostream(dctx);
}
if (dctx->file != NULL) {
isc_result_t tresult = ISC_R_UNSET;
tresult = closeandrename(dctx->f, result, dctx->tmpfile,
dctx->file);
if (tresult != ISC_R_SUCCESS && result == ISC_R_SUCCESS) {
@@ -1515,17 +1511,51 @@ dump_quantum(isc_task_t *task, isc_event_t *event) {
} else {
result = flushandsync(dctx->f, result, NULL);
}
dctx->result = result;
}
/*
* This will run in a network/task manager thread when the dump is complete.
*/
static void
master_dump_done_cb(void *data, isc_result_t result) {
dns_dumpctx_t *dctx = data;
if (result == ISC_R_SUCCESS && dctx->result != ISC_R_SUCCESS) {
result = dctx->result;
}
(dctx->done)(dctx->done_arg, result);
isc_event_free(&event);
dns_dumpctx_detach(&dctx);
}
/*
* This must be run from a network/task manager thread.
*/
static void
setup_dump(isc_task_t *task, isc_event_t *event) {
dns_dumpctx_t *dctx = NULL;
REQUIRE(isc_nm_tid() >= 0);
REQUIRE(event != NULL);
dctx = event->ev_arg;
REQUIRE(DNS_DCTX_VALID(dctx));
isc_nm_work_offload(isc_task_getnetmgr(task), master_dump_cb,
master_dump_done_cb, dctx);
isc_event_free(&event);
}
static isc_result_t
task_send(dns_dumpctx_t *dctx) {
isc_event_t *event;
event = isc_event_allocate(dctx->mctx, NULL, DNS_EVENT_DUMPQUANTUM,
dump_quantum, dctx, sizeof(*event));
setup_dump, dctx, sizeof(*event));
isc_task_send(dctx->task, &event);
return (ISC_R_SUCCESS);
}
@@ -1548,7 +1578,6 @@ dumpctx_create(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
dctx->done = NULL;
dctx->done_arg = NULL;
dctx->task = NULL;
dctx->nodes = 0;
dctx->first = true;
atomic_init(&dctx->canceled, false);
dctx->file = NULL;
@@ -1702,13 +1731,12 @@ writeheader(dns_dumpctx_t *dctx) {
}
static isc_result_t
dumptostreaminc(dns_dumpctx_t *dctx) {
dumptostream(dns_dumpctx_t *dctx) {
isc_result_t result = ISC_R_SUCCESS;
isc_buffer_t buffer;
char *bufmem;
dns_name_t *name;
dns_fixedname_t fixname;
unsigned int nodes;
isc_time_t start;
bufmem = isc_mem_get(dctx->mctx, initial_buffer_length);
@@ -1742,9 +1770,8 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
result = ISC_R_SUCCESS;
}
nodes = dctx->nodes;
isc_time_now(&start);
while (result == ISC_R_SUCCESS && (dctx->nodes == 0 || nodes--)) {
while (result == ISC_R_SUCCESS) {
dns_rdatasetiter_t *rdsiter = NULL;
dns_dbnode_t *node = NULL;
@@ -1780,52 +1807,7 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
result = dns_dbiterator_next(dctx->dbiter);
}
/*
* Work out how many nodes can be written in the time between
* two requests to the nameserver. Smooth the resulting number and
* use it as a estimate for the number of nodes to be written in the
* next iteration.
*/
if (dctx->nodes != 0 && result == ISC_R_SUCCESS) {
unsigned int pps = dns_pps; /* packets per second */
unsigned int interval;
uint64_t usecs;
isc_time_t end;
isc_time_now(&end);
if (pps < 100) {
pps = 100;
}
interval = 1000000 / pps; /* interval in usecs */
if (interval == 0) {
interval = 1;
}
usecs = isc_time_microdiff(&end, &start);
if (usecs == 0) {
dctx->nodes = dctx->nodes * 2;
if (dctx->nodes > 1000) {
dctx->nodes = 1000;
}
} else {
nodes = dctx->nodes * interval;
nodes /= (unsigned int)usecs;
if (nodes == 0) {
nodes = 1;
} else if (nodes > 1000) {
nodes = 1000;
}
/* Smooth and assign. */
dctx->nodes = (nodes + dctx->nodes * 7) / 8;
isc_log_write(dns_lctx, ISC_LOGCATEGORY_GENERAL,
DNS_LOGMODULE_MASTERDUMP,
ISC_LOG_DEBUG(1),
"dumptostreaminc(%p) new nodes -> %d",
dctx, dctx->nodes);
}
result = DNS_R_CONTINUE;
} else if (result == ISC_R_NOMORE) {
if (result == ISC_R_NOMORE) {
result = ISC_R_SUCCESS;
}
cleanup:
@@ -1835,11 +1817,11 @@ cleanup:
}
isc_result_t
dns_master_dumptostreaminc(isc_mem_t *mctx, dns_db_t *db,
dns_dbversion_t *version,
const dns_master_style_t *style, FILE *f,
isc_task_t *task, dns_dumpdonefunc_t done,
void *done_arg, dns_dumpctx_t **dctxp) {
dns_master_dumptostreamasync(isc_mem_t *mctx, dns_db_t *db,
dns_dbversion_t *version,
const dns_master_style_t *style, FILE *f,
isc_task_t *task, dns_dumpdonefunc_t done,
void *done_arg, dns_dumpctx_t **dctxp) {
dns_dumpctx_t *dctx = NULL;
isc_result_t result;
@@ -1855,7 +1837,6 @@ dns_master_dumptostreaminc(isc_mem_t *mctx, dns_db_t *db,
isc_task_attach(task, &dctx->task);
dctx->done = done;
dctx->done_arg = done_arg;
dctx->nodes = 100;
result = task_send(dctx);
if (result == ISC_R_SUCCESS) {
@@ -1881,7 +1862,7 @@ dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
return (result);
}
result = dumptostreaminc(dctx);
result = dumptostream(dctx);
INSIST(result != DNS_R_CONTINUE);
dns_dumpctx_detach(&dctx);
@@ -1927,11 +1908,11 @@ cleanup:
}
isc_result_t
dns_master_dumpinc(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
const dns_master_style_t *style, const char *filename,
isc_task_t *task, dns_dumpdonefunc_t done, void *done_arg,
dns_dumpctx_t **dctxp, dns_masterformat_t format,
dns_masterrawheader_t *header) {
dns_master_dumpasync(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
const dns_master_style_t *style, const char *filename,
isc_task_t *task, dns_dumpdonefunc_t done, void *done_arg,
dns_dumpctx_t **dctxp, dns_masterformat_t format,
dns_masterrawheader_t *header) {
FILE *f = NULL;
isc_result_t result;
char *tempname = NULL;
@@ -1956,7 +1937,6 @@ dns_master_dumpinc(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
isc_task_attach(task, &dctx->task);
dctx->done = done;
dctx->done_arg = done_arg;
dctx->nodes = 100;
dctx->file = file;
file = NULL;
dctx->tmpfile = tempname;
@@ -2001,7 +1981,7 @@ dns_master_dump(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
goto cleanup;
}
result = dumptostreaminc(dctx);
result = dumptostream(dctx);
INSIST(result != DNS_R_CONTINUE);
dns_dumpctx_detach(&dctx);
lib/dns/win32/libdns.def.in
+2 -2
View File
@@ -493,11 +493,11 @@ dns_lookup_cancel
dns_lookup_create
dns_lookup_destroy
dns_master_dump
dns_master_dumpinc
dns_master_dumpasync
dns_master_dumpnode
dns_master_dumpnodetostream
dns_master_dumptostream
dns_master_dumptostreaminc
dns_master_dumptostreamasync
dns_master_initrawheader
dns_master_loadbuffer
dns_master_loadbufferinc
lib/dns/zone.c
+1 -1
View File
@@ -2549,7 +2549,7 @@ zone_gotwritehandle(isc_task_t *task, isc_event_t *event) {
} else {
output_style = &dns_master_style_default;
}
result = dns_master_dumpinc(
result = dns_master_dumpasync(
zone->mctx, db, version, output_style, zone->masterfile,
zone->task, dump_done, zone, &zone->dctx,
zone->masterformat, &rawdata);