. . . . "VirtDumpRDFViewGraphs" . "2017-06-13T05:48:04Z" . . . . . "2017-06-13T05:48:04.955965"^^ . "2017-06-13T05:48:04.955965"^^ . "---+ Virtuoso Dump RDFView Graph Scripts\n\nThe RDF_QM_TREE_DUMP procedure and its associated procedures below are used for dumping one or more RDFView Graphs in a Virtuoso server to a set of turtle ttl dataset files in the specified dump directory. The dump generation is made as fast as possible by grouping mappings by underlying tables so many properties from neighbor database columns can be extracted in one table scan. The size of the generated files is limited to 5MB. The dump process creates internal stored procedures; their texts are saved in file .dump_procedures.sql in the directory of dump files for debugging purposes.\n\nNote that the dump directory must be included in the DirsAllowed parameter of the Virtuoso configuration file (e.g., virtuoso.ini), or the server will not be allowed to create nor access the dataset file(s).\n\nThe [[VirtBulkRDFLoader][Virtuoso RDF bulk loader]] scripts can then be used to load the dumped datasets for the RDFView graphs directly into a Virtuoso RDF QUAD store.\n\n---++ Parameters\n\n * in dest_dir VARCHAR - dump directory\n * in graph_iri VARCHAR - IRI of the graph to be dumped; triples from other graphs will be excluded. If NULL, then there's no restriction by graph.\n * in storage VARCHAR - IRI of the quad map storage to use. NULL means use default storage.\n * in root VARCHAR - IRI of the quad map to use, e.g., an IRI of an RDF View (or its part). NULL means use all RDF Views of the storage (and the default mapping as well).\n\n---++ Procedure Code\n\n\nCREATE PROCEDURE DB.DBA.RDF_QM_TREE_DUMP \n ( in dest_dir VARCHAR, \n in graph_iri VARCHAR := NULL, \n in storage VARCHAR := NULL, \n in root VARCHAR := NULL\n )\n{\n DECLARE all_qms, \n grouped_qmvs, \n launcher_text ANY;\n DECLARE grp_ctr, \n qm_ctr, \n qm_count INTEGER;\n DECLARE sql_file, \n launcher_name VARCHAR;\n IF (NOT (dest_dir LIKE '%/'))\n dest_dir := dest_dir || '/';\n sql_file := dest_dir || '.dump_procedures.sql';\n IF (storage IS NULL)\n storage := 'http://www.openlinksw.com/schemas/virtrdf#DefaultQuadStorage';\n string_to_file (\n sql_file, \n '-- This file contains procedure created by DB.DBA.RDF_QM_TREE_DUMP() for storage ' \n || COALESCE (storage, 'NULL') \n || ' and root quad map ' \n || COALESCE (root, 'NULL') \n || '\\n\\n', \n -2);\n all_qms := dict_list_keys (DB.DBA.RDF_QM_CONTENT_OF_QM_TREE (graph_iri, storage, root), 2);\n grouped_qmvs := DB.DBA.RDF_QM_GROUP_BY_SOURCE_TABLES (all_qms);\n launcher_name := 'RDF_QM_TREE_DUMP_BATCH_' || md5 (serialize (graph_iri) || storage || serialize (root));\n launcher_text := string_output ();\n http ('CREATE PROCEDURE DB.DBA.\"' || launcher_name || '\" (in dest_dir VARCHAR)\\n{\\n', launcher_text);\n FOR (grp_ctr := length (grouped_qmvs); grp_ctr > 0; grp_ctr := grp_ctr-2)\n {\n DECLARE tables, qms, proc_text ANY;\n DECLARE group_key, proc_name, dump_prefix, cmt VARCHAR;\n tables := grouped_qmvs [grp_ctr-2];\n qms := grouped_qmvs [grp_ctr-1];\n qm_count := length (qms);\n group_key := md5 (serialize (graph_iri) || storage || serialize (root) || serialize (tables));\n proc_name := 'RDF_QM_TREE_DUMP_GRP_' || group_key;\n proc_text := string_output ();\n cmt := sprintf ('%d quad maps on join of', qm_count);\n FOREACH (VARCHAR t IN tables) DO cmt := cmt || ' ' || t;\n http (' -- ' || cmt || '\\n', launcher_text);\n http (' DB.DBA.\"' || proc_name || '\" (dest_dir);\\n', launcher_text);\n http ('CREATE PROCEDURE DB.DBA.\"' || proc_name || '\" (in dest_dir VARCHAR)\\n', proc_text);\n http ('{\\n', proc_text);\n http (' -- ' || cmt || '\\n', proc_text);\n http (' DECLARE ses, env ANY;\\n', proc_text);\n http (' DECLARE file_ctr, cmt_len INTEGER;\\n', proc_text);\n http (' file_ctr := 0;\\n', proc_text);\n http (' dbg_obj_princ (' || WS.WS.STR_SQL_APOS (cmt) || ', '', file '', file_ctr);\\n', proc_text);\n http (' ses := string_output ();\\n', proc_text);\n http (' http (' || WS.WS.STR_SQL_APOS ('#' || cmt || '\\n') || ', ses);\\n', proc_text);\n http (' env := VECTOR (dict_new (16000), 0, '''', '''', '''', 0, 0, 0, 0);\\n', proc_text);\n http (' cmt_len := LENGTH (ses);\\n', proc_text);\n http (' FOR (SPARQL DEFINE input:storage <' || storage || '>\\n', proc_text);\n http (' SELECT ?s1, ?p1, ?o1\\n', proc_text);\n IF (graph_iri IS NOT NULL)\n {\n http (' WHERE { GRAPH <', proc_text); http_escape (graph_iri, 12, proc_text, 1, 1); http ('> {\\n', proc_text);\n }\n ELSE\n http (' WHERE { GRAPH ?g1 {\\n', proc_text);\n FOR (qm_ctr := 0; qm_ctr < qm_count; qm_ctr := qm_ctr + 1)\n {\n IF (qm_ctr > 0) http (' UNION\\n', proc_text);\n http (' { quad map <' || qms[qm_ctr] || '> { ?s1 ?p1 ?o1 } }\\n', proc_text);\n }\n http (' } } ) DO {\\n', proc_text);\n http (' http_ttl_triple (env, \"s1\", \"p1\", \"o1\", ses);\\n', proc_text);\n http (' IF (LENGTH (ses) > 5000000)\\n', proc_text);\n http (' {\\n', proc_text);\n http (' http ('' .\\\\n'', ses);\\n', proc_text);\n http (' string_to_file (sprintf (''%s' || group_key || '_%05d.ttl'', dest_dir, file_ctr), ses, -2);\\n', proc_text);\n http (' file_ctr := file_ctr + 1;\\n', proc_text);\n http (' dbg_obj_princ (' || WS.WS.STR_SQL_APOS (cmt) || ', '', file '', file_ctr);\\n', proc_text);\n http (' ses := string_output ();\\n', proc_text);\n http (' http (' || WS.WS.STR_SQL_APOS ('#' || cmt || '\\n') || ', ses);\\n', proc_text);\n http (' env := VECTOR (dict_new (16000), 0, '''', '''', '''', 0, 0, 0, 0);\\n', proc_text);\n http (' }\\n', proc_text);\n http (' }\\n', proc_text);\n http (' IF (LENGTH (ses) > cmt_len)\\n', proc_text);\n http (' {\\n', proc_text);\n http (' http ('' .\\\\n'', ses);\\n', proc_text);\n http (' string_to_file (sprintf (''%s' || group_key || '_%05d.ttl'', dest_dir, file_ctr), ses, -2);\\n', proc_text);\n http (' }\\n', proc_text);\n http ('}\\n', proc_text);\n proc_text := string_output_string (proc_text);\n string_to_file (sql_file, proc_text || ';\\n\\n' , -1);\n EXEC (proc_text);\n }\n http ('}\\n', launcher_text);\n launcher_text := string_output_string (launcher_text);\n string_to_file (sql_file, launcher_text || ';\\n\\n' , -1);\n EXEC (launcher_text);\n CALL ('DB.DBA.' || launcher_name)(dest_dir);\n}\n;\n\nCREATE FUNCTION DB.DBA.RDF_QM_CONTENT_OF_QM_TREE \n ( in graph_iri VARCHAR := NULL,\n in storage VARCHAR := NULL, \n in root VARCHAR := NULL, \n in dict ANY := NULL\n ) returns ANY\n{\n DECLARE res, subqms any;\n DECLARE graphiri varchar;\n graphiri := DB.DBA.JSO_SYS_GRAPH();\n IF (storage IS NULL)\n storage := 'http://www.openlinksw.com/schemas/virtrdf#DefaultQuadStorage';\n DB.DBA.RDF_QM_ASSERT_STORAGE_FLAG (storage, 0);\n IF (dict IS NULL)\n dict := dict_new ();\n IF (root IS NULL)\n {\n subqms := ((SELECT DB.DBA.VECTOR_AGG (sub.\"qmiri\")\n FROM (\n SPARQL DEFINE input:storage \"\"\n SELECT DISTINCT (str(?qm)) AS ?qmiri\n WHERE { GRAPH `iri(?:graphiri)` {\n { `iri(?:storage)` virtrdf:qsUserMaps ?lst .\n ?lst ?p ?qm .\n FILTER (0 = bif:strstr (str(?p), str(rdf:_)))\n } UNION {\n `iri(?:storage)` virtrdf:qsDefaultMap ?qm .\n } } } ) AS sub ) );\n FOREACH (varchar qmid IN subqms) DO\n DB.DBA.RDF_QM_CONTENT_OF_QM_TREE (graph_iri, storage, qmid, dict);\n RETURN dict;\n }\n DB.DBA.RDF_QM_ASSERT_JSO_TYPE (root, 'http://www.openlinksw.com/schemas/virtrdf#QuadMap');\n IF (graph_iri IS NOT NULL AND\n EXISTS ((SPARQL DEFINE input:storage \"\"\n SELECT (1) WHERE {\n GRAPH `iri(?:graphiri)` {\n `iri(?:root)` virtrdf:qmGraphRange-rvrFixedValue ?g .\n FILTER (str (?g) != str(?:graph_iri))\n } } ) ) )\n RETURN dict;\n IF (NOT EXISTS ((SPARQL DEFINE input:storage \"\"\n SELECT (1) WHERE {\n GRAPH `iri(?:graphiri)` {\n `iri(?:root)` virtrdf:qmMatchingFlags virtrdf:SPART_QM_EMPTY .\n } } ) ) )\n dict_put (dict, root, 1);\n subqms := ((SELECT DB.DBA.VECTOR_AGG (sub.\"qmiri\")\n FROM (\n SPARQL DEFINE input:storage \"\"\n SELECT DISTINCT (str(?qm)) as ?qmiri\n WHERE { GRAPH `iri(?:graphiri)` {\n \t\t`iri(?:root)` virtrdf:qmUserSubMaps ?lst .\n ?lst ?p ?qm .\n FILTER (0 = bif:strstr (str(?p), str(rdf:_)))\n } } ) AS sub ) );\n FOREACH (VARCHAR qmid IN subqms) DO\n DB.DBA.RDF_QM_CONTENT_OF_QM_TREE (graph_iri, storage, qmid, dict);\n RETURN dict;\n}\n;\n\nCREATE FUNCTION DB.DBA.RDF_QM_GROUP_BY_SOURCE_TABLES (in qms ANY) returns ANY\n{\n DECLARE res ANY;\n DECLARE ctr INTEGER;\n DECLARE graphiri VARCHAR;\n graphiri := DB.DBA.JSO_SYS_GRAPH();\n res := dict_new (LENGTH (qms) / 20);\n FOREACH (VARCHAR qmiri IN qms) DO\n {\n DECLARE tbls, acc ANY;\n tbls := ((SELECT DB.DBA.VECTOR_AGG (sub.\"tbl\")\n FROM (SELECT subsub.\"tbl\"\n FROM (\n SPARQL DEFINE input:storage \"\"\n SELECT DISTINCT ?tbl\n WHERE { GRAPH `iri(?:graphiri)` {\n { `iri(?:qmiri)` virtrdf:qmTableName ?tbl .\n } UNION {\n `iri(?:qmiri)` virtrdf:qmATables ?atbls .\n ?atbls ?p ?atbl .\n ?atbl virtrdf:qmvaTableName ?tbl\n } UNION {\n `iri(?:qmiri)` ?fldmap ?qmv .\n ?qmv virtrdf:qmvATables ?atbls .\n ?atbls ?p ?atbl .\n ?atbl virtrdf:qmvaTableName ?tbl .\n } } } ) subsub\n ORDER BY 1 ) AS sub ) );\n acc := dict_get (res, tbls);\n IF (acc IS NULL)\n vectorbld_init (acc);\n vectorbld_acc (acc, qmiri);\n dict_put (res, tbls, acc);\n }\n res := dict_to_vector (res, 2);\n FOR (ctr := LENGTH (res); ctr > 0; ctr := ctr-2)\n {\n DECLARE acc ANY;\n acc := aref_set_0 (res, ctr-1);\n vectorbld_final (acc);\n aset_zap_arg (res, ctr-1, acc);\n }\n RETURN res;\n}\n;\n\n--test dbg_obj_princ (DB.DBA.RDF_QM_GROUP_BY_SOURCE_TABLES (dict_list_keys (DB.DBA.RDF_QM_CONTENT_OF_QM_TREE (null), 2)));\n--test dbg_obj_princ (dict_list_keys (DB.DBA.RDF_QM_CONTENT_OF_QM_TREE (null), 2));\n--test DB.DBA.RDF_QM_TREE_DUMP ('dump/demo', null, null, null);\n--test DB.DBA.RDF_QM_TREE_DUMP ('dump/tpch', 'http://localhost:8600/tpch', null, null);\n\n\n\n---++ Example\n\n$ pwd\n/Applications/OpenLink Virtuoso/Virtuoso 6.1/database\n$ grep DirsAllowed virtuoso.iniDirsAllowed = ., ../vad, ./graphs\n$ /opt/virtuoso/bin/isql 1111\nConnected to OpenLink Virtuoso\nDriver: 06.01.3127 OpenLink Virtuoso ODBC Driver\nOpenLink Interactive SQL (Virtuoso), version 0.9849b.\nType HELP; for help and EXIT; to exit.\nSQL> DB.DBA.RDF_QM_TREE_DUMP('./graphs', 'http://opllinux5.usnet.private:8890/HR#', NULL, NULL);\n\nDone. -- 3297 msec.\nSQL> quit;\n$ ls -a graphs\n.\n..\n0501399cf19a391edea424b49f8fddec_00000.ttl\n0e92baa118032b03d45b671f8102d49b_00000.ttl\n1517d16a92affbb0918234caa9ee4506_00000.ttl\n1803645533d6b04b4ee1ba1d2b2ba517_00000.ttl\n21794e4c25ee4d3958cc7b92ea1ab2af_00000.ttl\n25e61545a84174e4f0e95b9dbd8400f1_00000.ttl\n2b3c3263a3ac32408124e3cf3119edb2_00000.ttl\n3d9bf6c213d70375e601125446a2e4b1_00000.ttl\n451ab264eb7b8ba653fab05ddb2fbb8f_00000.ttl\n47ac12f3099a1093359b91ffc141a4de_00000.ttl\n491406fea677f4663b68dd1c27d404bd_00000.ttl\n4f8d21f21e4ad7380f7da51b8b4750ba_00000.ttl\n5c6f12b92bb3b2f3601826ce7c66b4fa_00000.ttl\n7ad9ed11ef3557b3d11197037b672fce_00000.ttl\n7ae906be3dd4dd7198bf7bb9ebeb75c7_00000.ttl\n847132e12c95df9326b0f0c183c789cc_00000.ttl\n85da7768db9d91356ad56a4b2296ffeb_00000.ttl\n86ec197853e6e243b657593daadba07b_00000.ttl\n99227cc049c9b2cdf5f3c20278e85eb4_00000.ttl\na9d1085b8966bcbdbb3c1fc167e0bbe4_00000.ttl\naaef0bbd9f1172e795d41b9ee63575d4_00000.ttl\nafbb87dc2897a63491fe95d834e9bd3e_00000.ttl\nb5d8d72cab725914bff265c5766cc8e9_00000.ttl\nb7f574e8394e0408245a02c603618afe_00000.ttl\nc74baad55e1277b26fa002d070960b85_00000.ttl\nccd6ab4e49b6b3ac5b0e676e8b402639_00000.ttl\nd0e3534ac8ffa67e7e221ef6eec422ab_00000.ttl\nd1c409acf4379aa4e1d3e40b8bd1b059_00000.ttl\nde55244ca7b18e6a9ea7693b6275fb6c_00000.ttl\n.dump_procedures.sql\ne00e827d43678857097f706f66e050e4_00000.ttl\nebe38a29534ec94fbd5098014064b4d1_00000.ttl\nee7ff61ceb22dae7ae22b32cd99ace03_00000.ttl\nf87218215ec64026d7313e91f4c1ccbf_00000.ttl\n" . "VirtDumpRDFViewGraphs" . . . . . "VirtDumpRDFViewGraphs" . . "2017-06-13T05:48:04Z" . . "d261c85b5dd1f4a90de67cf8119639ed" .