1.1
1.2
1.3
1.4
1.4.1
1.4.2
1.4.3
1.4.4
1.4.5
1.4.6
1.4.7
1.4.8
1.4.9
1.5
1.5.1
1.5.2
1.5.3
1.6
1.6.1
1.6.2
1.6.3
1.6.4
1.6.5
1.6.6
1.7
1.7.1
1.7.2
1.7.3
1.7.3.1
1.7.3.2
1.7.4
1.7.5
1.7.6
1.7.7
1.8
1.8.1
1.8.2
1.8.3
1.8.4
1.8.4.1
1.9
TableofContentsIntroduction
ModellingDocumentInheritance
AccessingShapesData
AQL
UsingJoinsinAQL
UsingDynamicAttributeNames
CreatingTest-datausingAQL
DiffingDocuments
AvoidingParameterInjection
MultilineQueryStrings
Migratingnamedgraphfunctionsto3.0
Migratinganonymousgraphfunctionsto3.0
Migratinggraphmeasurementsto3.0
Graph
FulldepthGraph-Traversal
UsingacustomVisitor
ExampleAQLQueriesforGraphs
UseCases/Examples
CrawlingGithubwithPromises
UsingArangoDBwithSails.js
PopulatingaTextbox
ExportingData
AccessingbasedocumentswithJava
AddXMLdatatoArangoDBwithJava
Administration
UsingAuthentication
ImportingData
Replication
ReplicatingData
SlaveInitialization
XCopyInstallWindows
SilentNSISonWindows
Migrating2.8to3.0
Showgrantsfunction
Compiling/Build
CompileonDebian
CompileonWindows
OpenSSL
RunningCustomBuild
Recompilingjemalloc
Cloud,DCOSandDocker
1
1.9.1
1.9.2
1.9.3
1.9.4
1.9.5
1.9.6
1.9.7
1.9.8
1.9.9
1.10
1.10.1
1.10.2
1.10.3
1.10.4
RunningonAWS
UpdateonAWS
RunningonAzure
DockerArangoDB
DockerwithNodeJSApp
IntheGiantSwarm
ArangoDBinMesos
DC/OS:Fullexample
DC/OS:ChoosingContainerengine
Monitoring
Collectd-ReplicationSlaves
Collectd-Networkusage
Collectd-moreMetrics
Collectd-MonitoringFoxx
2
CookbookThiscookbookisfilledwithrecipestohelpyouunderstandthemulti-modeldatabaseArangoDBbetterandtohelpyouwithspecificproblems.
Youcanparticipateandwriteyourownrecipes.Youonlyneedtowritearecipeinmarkdownandmakeapullrequesttoourrepository.
Recipes
TherewillbesomesimplerecipestobringyouclosertoArangoDBandshowyoutheamountofpossibilitiesofourDatabase.TherealsowillbemorecomplexproblemstoshowyousolutiontospecificproblemsandthedepthofArangoDB.
Everyrecipeisdividedintothreeparts:
1. Problem:Adescriptionoftheproblem2. Solution:Adetailedsolutionofthegivenproblemwithcodeifanyisneeded3. Comment:Explanationofthesolution.Thispartisoptionaldependingonthecomplexityoftheproblem
Everyrecipehastagstoforabetteroverview:
#api,#aql,#arangosh,#collection,#database,#debian,#docker,#document,#driver,#foxx,#giantswarm,#graph,#howto,#java,#javascript,#join,#nodejs,#windows
Introduction
3
Modeldocumentinheritance
Problem
Howdoyoumodeldocumentinheritancegiventhatcollectionsdonotsupportthatfeature?
Solution
Letsassumeyouhavethreedocumentcollections:"subclass","class"and"superclass".Youalsohavetwoedgecollections:"sub_extends_class"and"class_extends_super".
Youcancreatethemviaarangoshorfoxx:
vargraph_module=require("com/arangodb/general-graph");
varg=graph_module._create("inheritance");
g._extendEdgeDefinitions(graph_module._directedRelation("sub_extends_class",["subclass"],["class"]));
g._extendEdgeDefinitions(graph_module._directedRelation("class_extends_super",["class"],["superclass"]));
Thismakessurewhenusingthegraphinterfacethattheinheritancelookslike:
sub→classclass→supersuper→sub
Tomakesureeverythingworksasexpectedyoushouldusethebuilt-intraversalincombinationwithFoxx.Thisallowsyoutoaddtheinheritancesecuritylayereasily.Tousetraversalsinfoxxsimplyaddthefollowinglinebeforedefiningroutes:
vartraversal=require("org/arangodb/graph/traversal");
varTraverser=traversal.Traverser;
AlsoyoucanaddthefollowingendpointinFoxx:
varreaderConfig={
datasource:traversal.graphDatasourceFactory("inheritance"),
expander:traversal.outboundExpander,//Goupwardsinthetree
visitor:function(config,result,vertex,path){
for(keyinvertex){
if(vertex.hasOwnProperty(key)&&!result.hasOwnProperty(key)){
result[key]=vertex[key]//Storeonlyattributesthathavenotyetbeenfound
}
}
}
};
controller.get("load/:collection/:key",function(req,res){
varresult={};
varid=res.params("collection")+"/"+res.params("key");
vartraverser=newTraverser(readerConfig);
traverser.traverse(result,g.getVertex(id));
res.json(result);
});
Thiswillmakesuretoiteratethecompleteinheritancetreeupwardstotherootelementandwillreturnallvaluesonthepathwerethefirstinstanceofthisvalueiskept
Comment
Youshouldgowithedgesbecauseitismucheasiertoquerythemifyouhaveatheoreticallyunlimiteddepthininheritance.IfyouhaveafixedinheritancedepthyoucouldalsogowithanattributeinthedocumentreferencingtheparentandexecutejoinsinAQL.
ModellingDocumentInheritance
4
Author:MichaelHackstein
Tags:#graph#document
ModellingDocumentInheritance
5
AccessingShapesData
Problem
Documentsinacollectionmayhavedifferentshapesassociatedwiththem.Thereisnowaytoquerytheshapesdatadirectly.Sohowdoyousolvethisproblem?
Solution
Therearetwopossiblewaystodothis.
A)Thefastwaywithsomerandomsamplings:
1. Askforarandomdocument(db.<collection>.any())andnoteitstop-levelattributenames2. Repeatthisforatleast10times.Afterthatrepeatitonlyifyouthinkit'sworthit.
Followingisanexampleofanimplementation:
attributes(db.myCollection);
functionattributes(collection){
"usestrict"
varprobes=10;
varmaxRounds=3;
varthreshold=0.5;
varmaxDocuments=collection.count();
if(maxDocuments<probes){
probes=maxDocuments;
}
if(probes===0){
return[];
}
varattributes={};
while(maxRounds--){
varnewDocuments=0;
varn=probes;
while(n--){
vardoc=collection.any();
varfound=false;
varkeys=Object.keys(doc);
for(vari=0;i<keys.length;++i){
if(attributes.hasOwnProperty(keys[i])){
++attributes[keys[i]];
}
else{
attributes[keys[i]]=1;
found=true;
}
}
if(found){
++newDocuments;
}
}
if(newDocuments/probes<=threshold){
break;
}
}
AccessingShapesData
6
returnObject.keys(attributes);
}
B)Thewaytofindalltop-levelattributes
Ifyoudon'tmindtomakesomeextrainsertsandyoudon'tcareaboutdeletionorupdatesofdocumentsyoucanusethefollowing:
db._create("mykeys");
db.mykeys.ensureUniqueSkiplist("attribute");
functioninsert(collection,document){
varresult=collection.save(document);
try{
varkeys=Objects.keys(document);
for(i=0;i<keys.length;++i){
try{
db.mykeys.save({attribute:keys[i]});
}
catch(err1){
//potentialuniquekeyconstraintviolations
}
}
}
catch(err2){
}
returnresult;
}
CommentA)Thefastwaywithsomerandomsamplings:
Yougetsomerandomsamplingwithboundedcomplexity.Ifyouhaveavarietyofattributesyoushouldrepeattheproceduremorethan10times.
Theprocedurecanbeimplementedasaserversideaction.
B)Thewaytofindalltop-levelattributes:
Thisprocedurewillnotcareaboutupdatesordeletionsofdocuments.Alsoonlythetop-levelattributeofthedocumentswillbeinsertedandnestedoneignored.
Theprocedurecanbeimplementedasaserversideaction.
Author:Arangodb
Tags:#collection#database
AccessingShapesData
7
AQL
UsingAQLingeneral
UsingJoinsinAQL
UsingDynamicAttributeNames
CreatingTest-datausingAQL
DiffingDocuments
AvoidingParameterInjection
MultilineQueryStrings
Migratingfrom2.xto3.0
Migratingnamedgraphfunctionsto3.0
Migratinganonymousgraphfunctionsto3.0
Migratinggraphmeasurementsto3.0
AQL
8
UsingJoinsinAQL
Problem
IwanttojoindocumentsfromcollectionsinanAQLquery.
One-to-Many:Ihaveacollectionusersandacollectioncities.AuserlivesinacityandIneedthecityinformationduringthequery.
Many-To-Many:Ihaveacollectionauthorsandbooks.Anauthorcanwritemanybooksandabookcanhavemanyauthors.Iwanttoreturnalistofbookswiththeirauthors.ThereforeIneedtojointheauthorsandbooks.
Solution
UnlikemanyNoSQLdatabases,ArangoDBdoessupportjoinsinAQLqueries.Thisissimilartothewaytraditionalrelationaldatabaseshandlethis.However,becausedocumentsallowformoreflexibility,joinsarealsomoreflexible.Thefollowingsectionsprovidesolutionsforcommonquestions.
One-To-Many
Youhaveacollectioncalledusers.Usersliveincityandacityisidentifiedbyitsprimarykey.Inprincipleyoucanembeddedthecitydocumentintotheusersdocumentandbehappywithit.
{
"_id":"users/2151975421",
"_key":"2151975421",
"_rev":"2151975421",
"name":{
"first":"John",
"last":"Doe"
},
"city":{
"name":"Metropolis"
}
}
Thisworkswellformanyusecases.Nowassume,thatyouhaveadditionalinformationaboutthecity,likethenumberofpeoplelivinginit.Itwouldbeimpracticaltochangeeachandeveryuserdocumentifthisnumberschanges.Thereforeitisgoodideatoholdthecityinformationinaseparatecollection.
arangosh>db.cities.document("cities/2241300989");
{
"population":1000,
"name":"Metropolis",
"_id":"cities/2241300989",
"_rev":"2241300989",
"_key":"2241300989"
}
Nowyouinsteadofembeddingthecitydirectlyintheuserdocument,youcanusethekeyofthecity.
arangosh>db.users.document("users/2290649597");
{
"name":{
"first":"John",
"last":"Doe"
},
"city":"cities/2241300989",
"_id":"users/2290649597",
"_rev":"2290649597",
"_key":"2290649597"
}
UsingJoinsinAQL
9
Wecannowjointhesetwocollectionsveryeasily.
arangosh>db._query(
........>"FORuINusers"+
........>"FORcINcities"+
........>"FILTERu.city==c._idRETURN{user:u,city:c}"
........>).toArray()
[
{
"user":{
"name":{
"first":"John",
"last":"Doe"
},
"city":"cities/2241300989",
"_id":"users/2290649597",
"_rev":"2290649597",
"_key":"2290649597"
},
"city":{
"population":1000,
"name":"Metropolis",
"_id":"cities/2241300989",
"_rev":"2241300989",
"_key":"2241300989"
}
}
]
UnlikeSQLthereisnospecialJOINkeyword.Theoptimizerensuresthattheprimaryindexisusedintheabovequery.
However,veryoftenitismuchmoreconvenientfortheclientofthequeryifasingledocumentwouldbereturned,wherethecityinformationisembeddedintheuserdocument-asinthesimpleexampleabove.WithAQLthereyoudonotneedtoforgothissimplification.
arangosh>db._query(
........>"FORuINusers"+
........>"FORcINcities"+
........>"FILTERu.city==c._idRETURNmerge(u,{city:c})"
........>).toArray()
[
{
"_id":"users/2290649597",
"_key":"2290649597",
"_rev":"2290649597",
"name":{
"first":"John",
"last":"Doe"
},
"city":{
"_id":"cities/2241300989",
"_key":"2241300989",
"_rev":"2241300989",
"population":1000,
"name":"Metropolis"
}
}
]
Soyoucanhaveboth:theconvenientrepresentationoftheresultforyourclientandtheflexibilityofjoinsforyourdatamodel.
Many-To-Many
Intherelationalwordyouneedathirdtabletomodelthemany-to-manyrelation.InArangoDByouhaveachoicedependingontheinformationyouaregoingtostoreandthetypeofquestionsyouaregoingtoask.
Assumethatauthorsarestoredinonecollectionandbooksinasecond.Ifallyouneedis"whicharetheauthorsofabook"thenyoucaneasilymodelthisasalistattributeinusers.
UsingJoinsinAQL
10
Ifyouwanttostoremoreinformation,forexamplewhichauthorwrotewhichpageinaconferenceproceeding,orifyoualsowanttoknow"whichbookswerewrittenbywhichauthor",youcanuseedgecollections.Thisisverysimilartothe"jointable"fromtherelationalworld.
EmbeddedLists
Ifyouonlywanttostoretheauthorsofabook,youcanembedthemaslistinthebookdocument.Thereisnoneedforaseparatecollection.
arangosh>db.authors.toArray()
[
{
"_id":"authors/2661190141",
"_key":"2661190141",
"_rev":"2661190141",
"name":{
"first":"Maxima",
"last":"Musterfrau"
}
},
{
"_id":"authors/2658437629",
"_key":"2658437629",
"_rev":"2658437629",
"name":{
"first":"John",
"last":"Doe"
}
}
]
Youcanquerybooks
arangosh>db._query("FORbINbooksRETURNb").toArray();
[
{
"_id":"books/2681506301",
"_key":"2681506301",
"_rev":"2681506301",
"title":"ThebeautyofJOINS",
"authors":[
"authors/2661190141",
"authors/2658437629"
]
}
]
andjointheauthorsinaverysimilarmannergivenintheone-to-manysection.
arangosh>db._query(
........>"FORbINbooks"+
........>"LETa=(FORxINb.authors"+
........>"FORaINauthorsFILTERx==a._idRETURNa)"+
........>"RETURN{book:b,authors:a}"
........>).toArray();
[
{
"book":{
"title":"ThebeautyofJOINS",
"authors":[
"authors/2661190141",
"authors/2658437629"
],
"_id":"books/2681506301",
"_rev":"2681506301",
"_key":"2681506301"
},
"authors":[
{
"name":{
UsingJoinsinAQL
11
"first":"Maxima",
"last":"Musterfrau"
},
"_id":"authors/2661190141",
"_rev":"2661190141",
"_key":"2661190141"
},
{
"name":{
"first":"John",
"last":"Doe"
},
"_id":"authors/2658437629",
"_rev":"2658437629",
"_key":"2658437629"
}
]
}
]
orembedtheauthorsdirectly
arangosh>db._query(
........>"FORbINbooksLETa=("+
........>"FORxINb.authors"+
........>"FORaINauthorsFILTERx==a._idRETURNa)"+
........>"RETURNmerge(b,{authors:a})"
........>).toArray();
[
{
"_id":"books/2681506301",
"_key":"2681506301",
"_rev":"2681506301",
"title":"ThebeautyofJOINS",
"authors":[
{
"_id":"authors/2661190141",
"_key":"2661190141",
"_rev":"2661190141",
"name":{
"first":"Maxima",
"last":"Musterfrau"
}
},
{
"_id":"authors/2658437629",
"_key":"2658437629",
"_rev":"2658437629",
"name":{
"first":"John",
"last":"Doe"
}
}
]
}
]
UsingEdgeCollections
Ifyoualsowanttoquerywhichbooksarewrittenbyagivenauthor,embeddingauthorsinthebookdocumentispossible,butitismoreefficienttouseaedgecollectionsforspeed.
Oryouarepublishingaproceeding,thenyouwanttostorethepagestheauthorhaswrittenaswell.Thisinformationcanbestoredintheedgedocument.
Firstcreatetheusers
arangosh>db._create("authors");
[ArangoCollection2926807549,"authors"(typedocument,statusloaded)]
arangosh>db.authors.save({name:{first:"John",last:"Doe"}})
UsingJoinsinAQL
12
{
"error":false,
"_id":"authors/2935261693",
"_rev":"2935261693",
"_key":"2935261693"
}
arangosh>db.authors.save({name:{first:"Maxima",last:"Musterfrau"}})
{
"error":false,
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813"
}
Nowcreatethebookswithoutanyauthorinformation.
arangosh>db._create("books");
[ArangoCollection2928380413,"books"(typedocument,statusloaded)]
arangosh>db.books.save({title:"ThebeautyofJOINS"});
{
"error":false,
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317"
}
Anedgecollectionisnowusedtolinkauthorsandbooks.
arangosh>db._createEdgeCollection("written");
[ArangoCollection2931132925,"written"(typeedge,statusloaded)]
arangosh>db.written.save("authors/2935261693",
........>"books/2980088317",
........>{pages:"1-10"})
{
"error":false,
"_id":"written/3006237181",
"_rev":"3006237181",
"_key":"3006237181"
}
arangosh>db.written.save("authors/2938210813",
........>"books/2980088317",
........>{pages:"11-20"})
{
"error":false,
"_id":"written/3012856317",
"_rev":"3012856317",
"_key":"3012856317"
}
InordertogetallbookswiththeirauthorsyoucanuseNEIGHBORS.
arangosh>db._query(
........>"FORbINbooksRETURN"+
........>"{"+
........>"book:b,"+
........>"authors:NEIGHBORS(books,"+
........>"written,"+
........>"b._id,"+
........>"'inbound'"+
........>")}"
........>).toArray();
[
{
"book":{
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317",
"title":"ThebeautyofJOINS"
UsingJoinsinAQL
13
},
"authors":[
{
"edge":{
"_id":"written/3006237181",
"_from":"authors/2935261693",
"_to":"books/2980088317",
"_rev":"3006237181",
"_key":"3006237181",
"pages":"1-10"
},
"vertex":{
"_id":"authors/2935261693",
"_rev":"2935261693",
"_key":"2935261693",
"name":{
"first":"John",
"last":"Doe"
}
}
},
{
"edge":{
"_id":"written/3012856317",
"_from":"authors/2938210813",
"_to":"books/2980088317",
"_rev":"3012856317",
"_key":"3012856317",
"pages":"11-20"
},
"vertex":{
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813",
"name":{
"first":"Maxima",
"last":"Musterfrau"
}
}
}
]
}
]
Orifyouwanttohidetheinformationstoredintheedge.
arangosh>db._query(
........>"FORbINbooksRETURN{"+
........>"book:b,authors:"+
........>"NEIGHBORS(books,written,b._id,'inbound')[*].vertex}"
........>).toArray();
[
{
"book":{
"title":"ThebeautyofJOINS",
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317"
},
"authors":[
{
"_id":"authors/2935261693",
"_rev":"2935261693",
"_key":"2935261693",
"name":{
"first":"John",
"last":"Doe"
}
},
{
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813",
"name":{
"first":"Maxima",
UsingJoinsinAQL
14
"last":"Musterfrau"
}
}
]
}
]
Oragainembedtheauthorsdirectlyintothebookdocument.
arangosh>db._query(
........>"FORbINbooksRETURNmerge("+
........>"b,"+
........>"{"+
........>"authors:"+
........>"NEIGHBORS(books,written,b._id,'inbound')[*].vertex})"
........>).toArray();
[
{
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317",
"title":"ThebeautyofJOINS",
"authors":[
{
"_id":"authors/2935261693",
"_rev":"2935261693",
"_key":"2935261693",
"name":{
"first":"John",
"last":"Doe"
}
},
{
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813",
"name":{
"first":"Maxima",
"last":"Musterfrau"
}
}
]
}
]
Ifyouneedtheauthorsandtheirbooks,simplyreversethedirection.
arangosh>db._query(
........>"FORaINauthorsRETURN"+
........>"merge(a,"+
........>"{books:NEIGHBORS(authors,written,a._id,'outbound')[*].vertex})"
........>).toArray();
[
{
"_id":"authors/2938210813",
"_rev":"2938210813",
"_key":"2938210813",
"name":{
"first":"Maxima",
"last":"Musterfrau"
},
"books":[
{
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317",
"title":"ThebeautyofJOINS"
}
]
},
{
"_id":"authors/2935261693",
"_rev":"2935261693",
UsingJoinsinAQL
15
"_key":"2935261693",
"name":{
"first":"John",
"last":"Doe"
},
"books":[
{
"_id":"books/2980088317",
"_rev":"2980088317",
"_key":"2980088317",
"title":"ThebeautyofJOINS"
}
]
}
]
Authors:FrankCeller
Tags:#join#aql
UsingJoinsinAQL
16
UsingdynamicattributenamesinAQL
Problem
IwantanAQLquerytoreturnresultswithattributenamesassembledbyafunction,orwithavariablenumberofattributes.
Thiswillnotworkbyspecifyingtheresultusingaregularobjectliteral,asobjectliteralsrequirethenamesandnumbersofattributestobefixedatquerycompiletime.
SolutionThereareseveralsolutionstogettingdynamicattributenamestowork.
Subquerysolution
Ageneralsolutionistoletasubqueryoranotherfunctiontoproducethedynamicattributenames,andfinallypassthemthroughtheZIP()functiontocreateanobjectfromthem.
Let'sassumewewanttoprocessthefollowinginputdocuments:
{"name":"test","gender":"f","status":"active","type":"user"}
{"name":"dummy","gender":"m","status":"inactive","type":"unknown","magicFlag":23}
Let'salsoassumeourgoalforeachofthesedocumentsistoreturnonlytheattributenamesthatcontainthelettera,togetherwiththeirrespectivevalues.
Toextracttheattributenamesandvaluesfromtheoriginaldocuments,wecanuseasubqueryasfollows:
LETdocuments=[
{"name":"test","gender":"f","status":"active","type":"user"},
{"name":"dummy","gender":"m","status":"inactive","type":"unknown","magicFlag":23}
]
FORdocINdocuments
RETURN(
FORnameINATTRIBUTES(doc)
FILTERLIKE(name,'%a%')
RETURN{
name:name,
value:doc[name]
}
)
Thesubquerywillonlyletattributenamespassthatcontainthelettera.Theresultsofthesubqueryarethenmadeavailabletothemainqueryandwillbereturned.Buttheattributenamesintheresultarestillnameandvalue,sowe'renotthereyet.
Solet'salsoemployAQL'sZIP()function,whichcancreateanobjectfromtwoarrays:
thefirstparametertoZIP()isanarraywiththeattributenamesthesecondparametertoZIP()isanarraywiththeattributevalues
Insteadofdirectlyreturningthesubqueryresult,wefirstcaptureitinavariable,andpassthevariable'snameandvaluecomponentsintoZIP()likethis:
LETdocuments=[
{"name":"test","gender":"f","status":"active","type":"user"},
{"name":"dummy","gender":"m","status":"inactive","type":"unknown","magicFlag":23}
]
FORdocINdocuments
LETattributes=(
UsingDynamicAttributeNames
17
FORnameINATTRIBUTES(doc)
FILTERLIKE(name,'%a%')
RETURN{
name:name,
value:doc[name]
}
)
RETURNZIP(attributes[*].name,attributes[*].value)
Notethatwehavetousetheexpansionoperator([*])onattributesbecauseattributesitselfisanarray,andwewanteitherthenameattributeorthevalueattributeofeachofitsmembers.
Toprovethisisworking,hereistheabovequery'sresult:
[
{
"name":"test",
"status":"active"
},
{
"name":"dummy",
"status":"inactive",
"magicFlag":23
}
]
Ascanbeseen,thetworesultshaveadifferentamountofresultattributes.Wecanalsomaketheresultabitmoredynamicbyprefixingeachattributewiththevalueofthenameattribute:
LETdocuments=[
{"name":"test","gender":"f","status":"active","type":"user"},
{"name":"dummy","gender":"m","status":"inactive","type":"unknown","magicFlag":23}
]
FORdocINdocuments
LETattributes=(
FORnameINATTRIBUTES(doc)
FILTERLIKE(name,'%a%')
RETURN{
name:CONCAT(doc.name,'-',name),
value:doc[name]
}
)
RETURNZIP(attributes[*].name,attributes[*].value)
Thatwillgiveusdocument-specificattributenameslikethis:
[
{
"test-name":"test",
"test-status":"active"
},
{
"dummy-name":"dummy",
"dummy-status":"inactive",
"dummy-magicFlag":23
}
]
Usingexpressionsasattributenames(ArangoDB2.5)
Ifthenumberofdynamicattributestoreturnisknowninadvance,andonlytheattributenamesneedtobecalculatedusinganexpression,thenthereisanothersolution.
ArangoDB2.5andhigherallowusingexpressionsinsteadoffixedattributenamesinobjectliterals.Usingexpressionsasattributenamesrequiresenclosingtheexpressioninextra[and]todisambiguatethemfromregular,unquotedattributenames.
UsingDynamicAttributeNames
18
Let'screatearesultthatreturnstheoriginaldocumentdatacontainedinadynamicallynamedattribute.We'llbeusingtheexpressiondoc.typefortheattributename.We'llalsoreturnsomeotherattributesfromtheoriginaldocuments,butprefixthemwiththedocuments'_keyattributevalues.Forthiswealsoneedattributenameexpressions.
Hereisaqueryshowinghowtodothis.Theattributenameexpressionsallrequiredtobeenclosedin[and]inordertomakethiswork:
LETdocuments=[
{"_key":"3231748397810","gender":"f","status":"active","type":"user"},
{"_key":"3231754427122","gender":"m","status":"inactive","type":"unknown"}
]
FORdocINdocuments
RETURN{
[doc.type]:{
[CONCAT(doc._key,"_gender")]:doc.gender,
[CONCAT(doc._key,"_status")]:doc.status
}
}
Thiswillreturn:
[
{
"user":{
"3231748397810_gender":"f",
"3231748397810_status":"active"
}
},
{
"unknown":{
"3231754427122_gender":"m",
"3231754427122_status":"inactive"
}
}
]
Note:attributenameexpressionsandregular,unquotedattributenamescanbemixed.
Author:JanSteemann
Tags:#aql
UsingDynamicAttributeNames
19
CreatingtestdatawithAQL
Problem
Iwanttocreatesometestdocuments.
Solution
Ifyouhaven'tyetcreatedacollectiontoholdthedocuments,createonenowusingtheArangoShell:
db._create("myCollection");
ThishascreatedacollectionnamedmyCollection.
OneoftheeasiestwaystofillacollectionwithtestdataistouseanAQLquerythatiteratesoverarange.
RunthefollowingAQLqueryfromtheAQLeditorinthewebinterfacetoinsert1,000documentsintothejustcreatedcollection:
FORiIN1..1000
INSERT{name:CONCAT("test",i)}INmyCollection
Thenumberofdocumentstocreatecanbemodifiedeasilybeadjustingtherangeboundaryvalues.
Tocreatemorecomplextestdata,adjusttheAQLquery!
Let'ssaywealsowantastatusattribute,andfillitwithintegervaluesbetween1to(including)5,withequaldistribution.Agoodwaytoachievethisistousethemodulooperator(%):
FORiIN1..1000
INSERT{
name:CONCAT("test",i),
status:1+(i%5)
}INmyCollection
Tocreatepseudo-randomvalues,usetheRAND()function.Itcreatespseudo-randomnumbersbetween0and1.Usesomefactortoscaletherandomnumbers,andFLOOR()toconvertthescalednumberbacktoaninteger.
Forexample,thefollowingquerypopulatesthevalueattributewithnumbersbetween100and150(including):
FORiIN1..1000
INSERT{
name:CONCAT("test",i),
value:100+FLOOR(RAND()*(150-100+1))
}INmyCollection
Afterthetestdatahasbeencreated,itisoftenhelpfultoverifyit.TheRAND()functionisalsoagoodcandidateforretrievingarandomsampleofthedocumentsinthecollection.Thisquerywillretrieve10randomdocuments:
FORdocINmyCollection
SORTRAND()
LIMIT10
RETURNdoc
TheCOLLECTclauseisaneasymechanismtorunanaggregateanalysisonsomeattribute.Let'ssaywewantedtoverifythedatadistributioninsidethestatusattribute.Inthiscasewecouldrun:
FORdocINmyCollection
COLLECTvalue=doc.valueWITHCOUNTINTOcount
RETURN{
CreatingTest-datausingAQL
20
value:value,
count:count
}
Theabovequerywillprovidethenumberofdocumentsperdistinctvalue.
Author:JanSteemann
Tags:#aql
CreatingTest-datausingAQL
21
DiffingTwoDocumentsinAQL
Problem
HowtocreateadiffofdocumentsinAQL
Solution
Thoughthereisnobuilt-inAQLfunctiontodifftwodocuments,itiseasilypossibletobuildyourownlikeinthefollowingquery:
/*inputdocument1*/
LETdoc1={
"foo":"bar",
"a":1,
"b":2
}
/*inputdocument2*/
LETdoc2={
"foo":"baz",
"a":2,
"c":3
}
/*collectattributespresentindoc1,butmissingindoc2*/
LETmissing=(
FORkeyINATTRIBUTES(doc1)
FILTER!HAS(doc2,key)
RETURN{
[key]:doc1[key]
}
)
/*collectattributespresentinbothdocs,butthathavedifferentvalues*/
LETchanged=(
FORkeyINATTRIBUTES(doc1)
FILTERHAS(doc2,key)&&doc1[key]!=doc2[key]
RETURN{
[key]:{
old:doc1[key],
new:doc2[key]
}
}
)
/*collectattributespresentindoc2,butmissingindoc1*/
LETadded=(
FORkeyINATTRIBUTES(doc2)
FILTER!HAS(doc1,key)
RETURN{
[key]:doc2[key]
}
)
/*returnfinalresult*/
RETURN{
"missing":missing,
"changed":changed,
"added":added
}
Note:Thequerymaylookabitlengthy,butmuchofthatisduetoformatting.Amoreterseversioncanbefoundbelow.
Theabovequerywillreturnadocumentwiththreeattributes:
missing:Containsallattributesonlypresentinfirstdocument(i.e.missinginseconddocument)
DiffingDocuments
22
changed:Containsallattributespresentinbothdocumentsthathavedifferentvaluesadded:Containsallattributesonlypresentinseconddocument(i.e.missinginfirstdocument)
Forthetwoexampledocumentsitwillreturn:
[
{
"missing":[
{
"b":2
}
],
"changed":[
{
"foo":{
"old":"bar",
"new":"baz"
}
},
{
"a":{
"old":1,
"new":2
}
}
],
"added":[
{
"c":3
}
]
}
]
Thatoutputformatwasthefirstthatcametomymind.Itisofcoursepossibletoadjustthequerysoitproducesadifferentoutputformat.
FollowingisaversionofthesamequerythatcanbeinvokedfromJavaScripteasily.Itpassesthetwodocumentsasbindparametersandcallsdb._query.Thequeryisnowanone-liner(lessreadablebuteasiertocopy&paste):
bindVariables={
doc1:{"foo":"bar","a":1,"b":2},
doc2:{"foo":"baz","a":2,"c":3}
};
query="LETdoc1=@doc1,doc2=@doc2,missing=(FORkeyINATTRIBUTES(doc1)FILTER!HAS(doc2,key)RETURN{[key]:doc1[
key]}),changed=(FORkeyINATTRIBUTES(doc1)FILTERHAS(doc2,key)&&doc1[key]!=doc2[key]RETURN{[key]:{old:doc1[
key],new:doc2[key]}}),added=(FORkeyINATTRIBUTES(doc2)FILTER!HAS(doc1,key)RETURN{[key]:doc2[key]})RETURN
{missing:missing,changed:changed,added:added}";
result=db._query(query,bindVariables).toArray();
Author:JanSteemann
Tags:#howto#aql
DiffingDocuments
23
AvoidingparameterinjectioninAQL
Problem
Idon'twantmyAQLqueriestobeaffectedbyparameterinjection.
Whatisparameterinjection?
Parameterinjectionmeansthatpotentiallycontentisinsertedintoaquery,andthatinjectionmaychangethemeaningofthequery.Itisasecurityissuethatmayallowanattackertoexecutearbitraryqueriesonthedatabasedata.
Itoftenoccursifapplicationstrustfullyinsertuser-providedinputsintoaquerystring,anddonotfullyorincorrectlyfilterthem.Italsooccursoftenwhenapplicationsbuildqueriesnaively,withoutusingsecuritymechanismsoftenprovidedbydatabasesoftwareorqueryingmechanisms.
ParameterinjectionexamplesAssemblingquerystringswithsimplestringconcatenationlookstrivial,butispotentiallyunsafe.Let'sstartwithasimplequerythat'sfedwithsomedynamicinputvalue,let'ssayfromawebform.AclientapplicationoraFoxxroutehappilypicksuptheinputvalue,andputsitintoaquery:
/*evil!*/
varwhat=req.params("searchValue");/*userinputvaluefromwebform*/
...
varquery="FORdocINcollectionFILTERdoc.value=="+what+"RETURNdoc";
db._query(query,params).toArray();
Theabovewillprobablyworkfinefornumericinputvalues.
Whatcouldanattackerdotothisquery?HereareafewsuggestionstouseforthesearchValueparameter:
forreturningalldocumentsinthecollection:1||trueforremovingalldocuments:1||trueREMOVEdocINcollection//forinsertingnewdocuments:1||trueINSERT{foo:"bar"}INcollection//
Itshouldhavebecomeobviousthatthisisextremelyunsafeandshouldbeavoided.
Anpatternoftenseentocounteractthisistryingtoquoteandescapepotentiallyunsafeinputvaluesbeforeputtingthemintoquerystrings.Thismayworkinsomesituations,butit'seasytooverlooksomethingorgetitsubtlywrong:
/*we'resanitzingnow,butit'sstillevil!*/
varvalue=req.params("searchValue").replace(/'/g,'');
...
varquery="FORdocINcollectionFILTERdoc.value=='"+value+"'RETURNdoc";
db._query(query,params).toArray();
Theaboveexampleusessinglequotesforenclosingthepotentiallyunsafeuserinput,andalsoreplacesallsinglequotesintheinputvaluebeforehand.Notonlymaythatchangetheuserinput(leadingtosubtleerrorssuchas"whydoesmysearchforO'Briendon'treturnanyresults?"),butitisalsounsafe.Iftheuserinputcontainsabackslashattheend(e.g.foobar\),thatbackslashwillescapetheclosingsinglequote,allowingtheuserinputtobreakoutofthestringfenceagain.
Itgetsworseifuserinputisinsertedintothequeryatmultipleplaces.Let'sassumewehaveaquerywithtwodynamicvalues:
query="FORdocINcollectionFILTERdoc.value=='"+value+"'&&doc.type=='"+type+"'RETURNdoc";
Ifanattackerinserted\forparametervalueand||trueREMOVEdocINcollection//forparametertype,thentheeffectivequerywouldbecome
AvoidingParameterInjection
24
FORdocINcollectionFILTERdoc.value=='\'&&doc.type=='||trueREMOVEdocINcollection//'RETURNdoc
whichishighlyundesirable.
Solution
Insteadofmixingquerystringfragmentswithuserinputsnaivelyviastringconcatenation,useeitherbindparametersoraquerybuilder.Bothcanhelptoavoidtheproblemofinjection,becausetheyallowseparatingtheactualqueryoperations(likeFOR,INSERT,REMOVE)from(userinput)values.
Thisrecipefocusesonusingbindparameters.Thisisnottosaythatquerybuildersshouldn'tbeused.Theyweresimplyomittedhereforthesakeofsimplicity.TogetstartedwithausinganAQLquerybuilderinArangoDBorotherJavaScriptenvironments,havealookataqb(whichcomesbundledwithArangoDB).InsideArangoDB,therearealsoFoxxquerieswhichcanbecombinedwithaqb.
Whatbindparametersare
BindparametersinAQLqueriesarespecialtokensthatactasplaceholdersforactualvalues.Here'sanexample:
FORdocINcollection
FILTERdoc.value==@what
RETURNdoc
Intheabovequery,@whatisabindparameter.Inordertoexecutethisquery,avalueforbindparameter@whatmustbespecified.Otherwisequeryexecutionwillfailwitherror1551(novaluespecifiedfordeclaredbindparameter).Ifavaluefor@whatgetsspecified,thequerycanbeexecuted.However,thequerystringandthebindparametervalues(i.e.thecontentsofthe@whatbindparameter)willbehandledseparately.What'sinthebindparameterwillalwaysbetreatedasavalue,anditcan'tgetoutofitssandboxandchangethesemanticmeaningofaquery.
Howbindparametersareused
Toexecuteaquerywithbindparameters,thequerystring(containingthebindparameters)andthebindparametervaluesarespecifiedseparately(notethatwhenthebindparametervalueisassigned,theprefix@needstobeomitted):
/*querystringwithbindparameter*/
varquery="FORdocINcollectionFILTERdoc.value==@whatRETURNdoc";
/*actualvalueforbindparameter*/
varparams={what:42};
/*runquery,specifyingquerystringandbindparameterseparately*/
db._query(query,params).toArray();
Ifamalicioususerwouldset@whattoavalueof1||true,thiswouldn'tdoanyharm.AQLwouldtreatthecontentsof@whatasasinglestringtoken,andthemeaningofthequerywouldremainunchanged.Theactuallyexecutedquerywouldbe:
FORdocINcollection
FILTERdoc.value=="1||true"
RETURNdoc
Thankstobindparametersitisalsoimpossibletoturnaselection(i.e.read-only)queryintoadatadeletionquery.
UsingJavaScriptvariablesasbindparameters
Thereisalsoatemplatestringgeneratorfunctionaqlthatcanbeusedtosafely(andconveniently)builtAQLqueriesusingJavaScriptvariablesandexpressions.Itcanbeinvokedasfollows:
constaql=require('@arangodb')aql;//notneededinarangosh
varvalue="someinputvalue";
AvoidingParameterInjection
25
varquery=aql`FORdocINcollection
FILTERdoc.value==${value}
RETURNdoc`;
varresult=db._query(query).toArray();
NotethatanES6templatestringisusedforpopulatingthequeryvariable.ThestringisassembledusingtheaqlgeneratorfunctionwhichisbundledwithArangoDB.ThetemplatestringcancontainreferencestoJavaScriptvariablesorexpressionsvia${...}.Intheaboveexample,thequeryreferencesavariablenamedvalue.Theaqlfunctiongeneratesanobjectwithtwoseparateattributes:thequerystring,containingreferencestobindparameters,andtheactualbindparametervalues.
Bindparameternamesareautomaticallygeneratedbytheaqlfunction:
varvalue="someinputvalue";
aql`FORdocINcollectionFILTERdoc.value==${value}RETURNdoc`;
{
"query":"FORdocINcollectionFILTERdoc.value==@value0RETURNdoc",
"bindVars":{
"value0":"someinputvalue"
}
}
Usingbindparametersindynamicqueries
Bindparametersarehelpful,soitmakessensetousethemforhandlingthedynamicvalues.Youcanevenusethemforqueriesthatitselfarehighlydynamic,forexamplewithconditionalFILTERandLIMITparts.Here'showtodothis:
/*note:thisexamplehasaslightissue...hangonreading*/
varquery="FORdocINcollection";
varparams={};
if(useFilter){
query+="FILTERdoc.value==@what";
params.what=req.params("searchValue");
}
if(useLimit){
/*notquiteright,seebelow*/
query+="LIMIT@offset,@count";
params.offset=req.params("offset");
params.count=req.params("count");
}
query+="RETURNdoc";
db._query(query,params).toArray();
Notethatinthisexamplewe'rebacktostringconcatenation,butwithouttheproblemofthequerybeingvulnerabletoarbitrarymodifications.
Inputvaluevalidationandsanitation
Stillyoushouldprefertobeparanoid,andtrytodetectinvalidinputvaluesasearlyaspossible,atleastbeforeexecutingaquerywiththem.Thisisbecausesomeinputparametersmayaffecttheruntimebehaviorofqueriesnegativelyor,whenmodified,mayleadtoqueriesthrowingruntimeerrorsinsteadofreturningvalidresults.Thisisn'tsomethinganattackershoulddeserve.
LIMITisagoodexampleforthis:ifusedwithasingleargument,theargumentshouldbenumeric.WhenLIMITisgivenastringvalue,executingthequerywillfail.Youmaywanttodetectthisearlyanddon'treturnanHTTP500(asthiswouldsignalattackersthattheyweresuccessfulbreakingyourapplication).
AnotherproblemwithLIMITisthathighLIMITvaluesarelikelymoreexpensivethanlowones,andyoumaywanttodisallowusingLIMITvaluesexceedingacertainthreshold.
Here'swhatyoucoulddoinsuchcases:
varquery="FORdocINcollectionLIMIT@countRETURNdoc";
AvoidingParameterInjection
26
/*somedefaultvalueforlimit*/
varparams={count:100};
if(useLimit){
varcount=req.params("count");
/*abortifvaluedoesnotlooklikeaninteger*/
if(!preg_match(/^d+$/,count)){
throw"invalidcountvalue!";
}
/*actuallyturnitintoaninteger*/
params.count=parseInt(count,10);//turnintonumericvalue
}
if(params.count<1||params.count>1000){
/*valueisoutsideofacceptedthresholds*/
throw"invalidcountvalue!";
}
db._query(query,params).toArray();
Thisisabitmorecomplex,butthat'sapriceyou'relikelywillingtopayforabitofextrasafety.Inrealityyoumaywanttouseaframeworkforvalidation(suchasjoiwhichcomesbundledwithArangoDB)insteadofwritingyourownchecksallovertheplace.
Bindparametertypes
TherearetwotypesofbindparametersinAQL:
bindparametersforvalues:thoseareprefixedwithasingle@inAQLqueries,andarespecifiedwithouttheprefixwhentheygettheirvalueassigned.ThesebindparameterscancontainanyvalidJSONvalue.
Examples:@what,@searchValue
bindparametersforcollections:theseareprefixedwith@@inAQLqueries,andarereplacedwiththenameofacollection.Whenthebindparametervalueisassigned,theparameteritselfmustbespecifiedwithasingle@prefix.Onlystringvaluesareallowedforthistypeofbindparameters.
Examples:@@collection
Thelattertypeofbindparameterisprobablynotusedasoften,anditshouldnotbeusedtogetherwithuserinput.OtherwiseusersmayfreelydetermineonwhichcollectionyourAQLquerieswilloperate(note:thismaybeavalidusecase,butnormallyitisextremelyundesired).
Authors:JanSteemann
Tags:#injection#aql#security
AvoidingParameterInjection
27
Writingmulti-lineAQLqueries
Problem
IwanttowriteanAQLquerythatspansmultiplelinesinmyJavaScriptsourcecode,butitdoesnotwork.Howtodothis?
Solution
AQLsupportsmulti-linequeries,andtheAQLeditorinArangoDB'swebinterfacesupportsthemtoo.
Whenissuedprogrammatically,multi-linequeriescanbeasourceoferrors,atleastinsomelanguages.Forexample,JavaScriptisnotoriouslybadathandlingmulti-line(JavaScript)statements,anduntilrecentlyithadnosupportformulti-linestrings.
InJavaScript,therearethreewaysofwritingamulti-lineAQLqueryinthesourcecode:
stringconcatenationES6templatestringsquerybuilder
Whichmethodworksbestdependsonafewfactors,butisoftenenoughasimplematterofpreference.Beforedecidingonany,pleasemakesuretoreadtherecipeforavoidingparameterinjectiontoo.
Stringconcatenation
WewantthequeryFORdocINcollectionFILTERdoc.value==@whatRETURNdoctobecomemorelegibleinthesourcecode.
SimplysplittingthequerystringintothreelineswillleaveuswithaparseerrorinJavaScript:
/*willnotwork*/
varquery="FORdocINcollection
FILTERdoc.value==@what
RETURNdoc";
Instead,wecoulddothis:
varquery="FORdocINcollection"+
"FILTERdoc.value==@what"+
"RETURNdoc";
ThisisperfectlyvalidJavaScript,butit'serror-prone.Peoplehavespentagesonfindingsubtlebugsintheirqueriesbecausetheymissedasinglewhitespacecharacteratthebeginningorstartofsomeline.
Pleasenotethatwhenassemblingqueriesviastringconcatenation,youshouldstillusebindparameters(asdoneabovewith@what)andnotinsertuserinputvaluesintothequerystringwithoutsanitation.
ES6templatestrings
ES6templatestringsareeasiertogetrightandalsolookmoreelegant.TheycanbeusedinsideArangoDBsinceversion2.5.butsomeotherplatformsdon'tsupportthemet.Forexample,theycan'tbeusedinIEandoldernode.jsversions.Sousethemifyourenvironmentsupportsthemandyourcodedoesnotneedtorunonanynon-ES6environments.
Here'sthequerystringdeclaredviaanES6templatestring(notethatthestringmustbeenclosedinbackticksnow):
varquery=`FORdocINcollection
FILTERdoc.value==@what
RETURNdoc`;
Thewhitespaceinthetemplatestring-variantismucheasiertogetrightthanwhendoingthestringconcatenation.
MultilineQueryStrings
28
Thereareafewthingstonoteregardingtemplatestrings:
ES6templatestringscanbeusedtoinjectJavaScriptvaluesintothestringdynamically.Substitutionsstartwiththecharactersequence${.CaremustbetakenifthissequenceitselfisusedinsidetheAQLquerystring(currentlythiswouldbeinvalidAQL,butthismaychangeinfutureArangoDBversions).Additionally,anyvaluesinjectedintothequerystringusingparametersubstitutionswillnotbeescapedcorrectlyautomatically,soagainspecialcaremustbetakenwhenusingthismethodtokeepqueriessafefromparameterinjection.
amulti-linetemplatestringwillactuallycontainnewlinecharacters.Thisisnotnecessarilythecasewhendoingstringconcatenation.Inthestringconcatenationexample,weusedthreelinesofsourcecodetocreateasingle-linequerystring.Wecouldhaveinsertednewlinesintothequerystringtheretoo,butwedidn't.Justtopointoutthatthetwovariantswillnotcreatebytewise-identicalquerystrings.
PleasenotethatwhenusingES6templatestringsforyourqueries,youshouldstillusebindparameters(asdoneabovewith@what)andnotinsertuserinputvaluesintothequerystringwithoutsanitation.
ThereisaconveniencefunctionaqlwhichcanbeusedtosafelyandeasilybuildanAQLquerywithsubstitutionsfromarbitraryJavaScriptvaluesandexpressions.Itcanbeinvokedlikethis:
constaql=require("@arangodb").aql;//notneededinarangosh
varwhat="someinputvalue";
varquery=aql`FORdocINcollection
FILTERdoc.value==${what}
RETURNdoc`;
Thetemplatestringvariantthatusesaqlisbothconvenientandsafe.Internally,itwillturnthesubstitutedvaluesintobindparameters.Thequerystringandthebindparametervalueswillbereturnedseparately,sotheresultofqueryabovewillbesomethinglike:
{
"query":"FORdocINcollectionFILTERdoc.value==@value0RETURNdoc",
"bindVars":{
"value0":"someinputvalue"
}
}
Querybuilder
ArangoDBcomesbundledwithaquerybuildernamedaqb.ThatquerybuildercanbeusedtoprogrammaticallyconstructAQLqueries,withouthavingtowritequerystringsatall.
Here'sanexampleofitsusage:
varqb=require("aqb");
varjobs=db._createStatement({
query:(
qb.for('job').in('_jobs')
.filter(
qb('pending').eq('job.status')
.and(qb.ref('@queue').eq('job.queue'))
.and(qb.ref('@now').gte('job.delayUntil'))
)
.sort('job.delayUntil','ASC')
.limit('@max')
.return('job')
),
bindVars:{
queue:queue._key,
now:Date.now(),
max:queue.maxWorkers-numBusy
}
}).execute().toArray();
MultilineQueryStrings
29
Ascanbeseen,aqbprovidesafluentAPIthatallowschainingfunctioncallsforcreatingtheindividualqueryoperations.Thishasafewadvantages:
flexibility:thereisnoquerystringinthesourcecode,sothecodecanbeformattedasdesiredwithouthavingtobotheraboutstringsvalidation:thequerycanbevalidatedsyntacticallybyaqbbeforebeingactuallyexecutedbytheserver.Testingofqueriesalsobecomeseasier.Additionally,someIDEsmayprovideauto-completiontosomeextendandthusaiddevelopmentsecurity:built-inseparationofqueryoperations(e.g.FOR,FILTER,SORT,LIMIT)anddynamicvalues(e.g.userinputvalues)
aqbcanbeusedinsideArangoDBandfromnode.jsandevenfromwithinbrowsers.
Authors:JanSteemann
Tags:#aql#aqb#es6
MultilineQueryStrings
30
MigratingGRAPH_*Functionsfrom2.8orearlierto3.0
Problem
Withthereleaseof3.0allGRAPHfunctionshavebeendroppedfromAQLinfavorofamorenativeintegrationofgraphfeaturesintothequerylanguage.Ihaveusedtheoldgraphfunctionsandwanttoupgradeto3.0.
Graphfunctionscoveredinthisrecipe:
GRAPH_COMMON_NEIGHBORSGRAPH_COMMON_PROPERTIESGRAPH_DISTANCE_TOGRAPH_EDGESGRAPH_NEIGHBORSGRAPH_TRAVERSALGRAPH_TRAVERSAL_TREEGRAPH_SHORTEST_PATHGRAPH_PATHSGRAPH_VERTICES
Solution1:QuickandDirty(notrecommended)
Whentousethissolution
Iamnotwillingtoinvestalotiftimeintotheupgradeprocessandiamwillingtosurrendersomeperformanceinfavoroflesseffort.Someconstellationsmaynotworkwiththissolutionduetothenatureofuser-definedfunctions.EspeciallycheckforAQLqueriesthatdobothmodificationsandGRAPH_*functions.
Registeringuser-definedfunctions
ThisstephastobeexecutedonceonArangoDBforeverydatabaseweareusing.
Weconnecttoarangodbwitharangoshtoissuethefollowingcommandstwo:
vargraphs=require("@arangodb/general-graph");
graphs._registerCompatibilityFunctions();
ThesehaveregisteredalloldGRAPH_*functionsasuser-definedfunctionsagain,withtheprefixarangodb::.
Modifytheapplicationcode
NextwehavetogothroughourapplicationcodeandreplaceallcallstoGRAPH_*byarangodb::GRAPH_*.Nowrunatestrunofourapplicationandcheckifitworked.Ifitworkedwearereadytogo.
ImportantInformation
Theuserdefinedfunctionswillcalltranslatedsubqueries(asdescribedinSolution2).Theoptimizerdoesnotknowanythingaboutthesesubqueriesbeforehandandcannotoptimizethewholeplan.Alsotheremightberead/writeconstellationsthatareforbiddeninuser-definedfunctions,thereforea"really"translatedquerymayworkwhiletheuser-definedfunctionworkaroundmayberejected.
Solution2:Translatingthequeries(recommended)
Whentousethissolution
Migratingnamedgraphfunctionsto3.0
31
Iamwillingtoinvestsometimeonmyqueriesinordertogetmaximumperformance,fullqueryoptimizationandabettercontrolofmyqueries.Noforcingintotheoldlayoutanymore.
Beforeyoustart
IfyouareusingvertexExampleswhicharenotonly_idstringsdonotskiptheGRAPH_VERTICESsection,becauseitwilldescribehowtotranslatethemtoAQL.AllgraphfunctionsusingavertexExampleareidenticaltoexecutingaGRAPH_VERTICESbeforeandusingit'sresultasstartpoint.ExamplewithNEIGHBORS:
FORresINGRAPH_NEIGHBORS(@graph,@myExample)RETURNres
Isidenticalto:
FORstartGRAPH_VERTICES(@graph,@myExample)
FORresINGRAPH_NEIGHBORS(@graph,start)RETURNres
AllnonGRAPH_VERTICESfunctionswillonlyexplainthetransformationforasingleinputdocument's_id.
Optionsusedeverywhere
OptionedgeCollectionRestriction
InordertouseedgeCollectionrestrictionwejustusethefeaturethatthetraversercanwalkoveralistofedgecollectionsdirectly.SotheedgeCollectionRestrictionsjustformthislist(exampleGraphEdges):
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId,{edgeCollectionRestriction:[edges1,edges2]})RETURNe
//NEW
[..]FORv,eINANY@startIdedges1,edges2RETURNDISTINCTe._id
Note:The@graphNamebindParameterisnotusedanymoreandprobablyhastoberemovedfromthequery.
OptionincludeData
IfweusetheoptionincludeDatawesimplyreturntheobjectdirectlyinsteadofonlythe_id
ExampleGRAPH_EDGES:
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId,{includeData:true})RETURNe
//NEW
[..]FORv,eINANY@startIdGRAPH@graphNameRETURNDISTINCTe
Optiondirection
Thedirectionhastobeplacedbeforethestartid.Notehere:ThedirectionhastobeplacedasWorditcannotbehandedinviaabindParameteranymore:
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId,{direction:'inbound'})RETURNe
//NEW
[..]FORv,eININBOUND@startIdGRAPH@graphNameRETURNDISTINCTe._id
OptionsminDepth,maxDepth
IfweusetheoptionsminDepthandmaxDepth(bothdefault1ifnotset)wecansimplyputtheminfrontofthedirectionpartintheTraversalstatement.
Migratingnamedgraphfunctionsto3.0
32
ExampleGRAPH_EDGES:
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId,{minDepth:2,maxDepth:4})RETURNe
//NEW
[..]FORv,eIN2..4ANY@startIdGRAPH@graphNameRETURNDISTINCTe._id
OptionmaxIteration
TheoptionmaxIterationsisremovedwithoutreplacement.Yourqueriesarenowboundbymainmemorynotbyanarbitrarynumberofiterations.
GRAPH_VERTICES
Firstwehavetobranchontheexample.Therewehavethreepossibilities:
1. Theexampleisan_idstring.2. Theexampleisnullor{}.3. Theexampleisanonemptyobjectoranarray.
Exampleis'_id'string
Thisistheeasiestreplacement.InthiscasewesimplyreplacethefunctionwithacalltoDOCUMENT:
//OLD
[..]GRAPH_VERTICES(@graphName,@idString)[..]
//NEW
[..]DOCUMENT(@idString)[..]
NOTE:The@graphNameisnotrequiredanymore,wemayhavetoadjustbindParameters.
TheAQLgraphfeaturescanworkwithaniddirectly,noneedtocallDOCUMENTbeforeifwejustneedthistofindastartingpoint.
Exampleisnullortheemptyobject
Thiscasemeansweusealldocumentsfromthegraph.Herewefirsthavetonowthevertexcollectionsofthegraph.
1. Ifweonlyhaveonecollection(sayvertices)wecanreplaceitwithasimpleiterationoverthiscollection:
//OLD
[..]FORvINGRAPH_VERTICES(@graphName,{})[..]
//NEW
[..]FORvINvertices[..]
`
NOTE:The@graphNameisnotrequiredanymore,wemayhavetoadjustbindParameters.
1. Wehavemorethanonecollection.Thisistheunfortunatecaseforageneralreplacement.Sointhegeneralreplacementweassumewedonotwanttoexcludeanyofthecollectionsinthegraph.ThanweunfortunatelyhavetoformaUNIONoverallthesecollections.Sayourgraphhasthevertexcollectionsvertices1,vertices2,vertices3wecreateasub-queryforasinglecollectionforeachofthemandwraptheminacalltoUNION.
//OLD
[..]FORvINGRAPH_VERTICES(@graphName,{})[..]
//NEW
[..]
FORvINUNION(//WestartaUNION
(FORvINvertices1RETURNv),//Foreachvertexcollection
(FORvINvertices2RETURNv),//wecreatethesamesubquery
(FORvINvertices3RETURNv)
Migratingnamedgraphfunctionsto3.0
33
)//FinishwiththeUNION
[..]
`
NOTE:IfyouhaveanymoredomainknowledgeofyourgraphapplyitatthispointtoidentifywhichcollectionsareactuallyrelevantasthisUNIONisaratherexpensiveoperation.
IfweusetheoptionvertexCollectionRestrictionintheoriginalquery.TheUNIONhastobeformedbythecollectionsinthisrestrictioninsteadofALLcollections.
Exampleisanon-emptyobject
Firstwefollowtheinstructionsfortheemptyobjectabove.Inthissectionwewilljustfocusonasinglecollectionvertices,theUNIONformultiplecollectionsisagainwrappedaroundasubqueryforeachofthesecollectionsbuiltinthefollowingway.
NowwehavetotransformtheexampleintoanAQLFILTERstatement.Thereforewetakealltop-levelattributesoftheexampleanddoanequalcomparisonwiththeirvalues.AllofthesecomparisonsarejoinedwithanANDbecausetheallhavetobefulfilled.
Example:
//OLD
[..]FORvINGRAPH_VERTICES(@graphName,{foo:'bar',the:{answer:42}}})[..]
//NEW
[..]FORvINvertices
FILTERv.foo=='bar'//foo:bar
ANDv.the=={answer:42}//the:{answer:42}
[..]
Exampleisanarray
Theideatransformationisalmostidenticaltoasinglenon-emptyobject.ForeachelementinthearraywecreatethefilterconditionsandthanweOR-combinethem(mindthebrackets):
//OLD
[..]FORvINGRAPH_VERTICES(@graphName,[{foo:'bar',the:{answer:42}},{foo:'baz'}]))[..]
//NEW
[..]FORvINvertices
FILTER(v.foo=='bar'//foo:bar
ANDv.the=={answer:42})//the:{answer:42}
OR(v.foo=='baz')
[..]
GRAPH_EDGES
TheGRAPH_EDGEScanbesimplyreplacedbyacalltotheAQLtraversal.
Nooptions
ThedefaultoptionsdiduseadirectionANYandreturnedadistinctresultoftheedges.Alsoitdidjustreturntheedges_idvalue.
//OLD
[..]FOReINGRAPH_EDGES(@graphName,@startId)RETURNe
//NEW
[..]FORv,eINANY@startIdGRAPH@graphNameRETURNDISTINCTe._id
OptionedgeExamples.
SeeGRAPH_VERTICESonhowtotransformexamplestoAQLFILTER.Applythefilterontheedgevariablee.
GRAPH_NEIGHBORS
Migratingnamedgraphfunctionsto3.0
34
TheGRAPH_NEIGHBORSisabreadth-first-searchonthegraphwithaglobaluniquecheckforvertices.SowecanreplaceitbyaanAQLtraversalwiththeseoptions.
Nooptions
ThedefaultoptionsdiduseadirectionANYandreturnedadistinctresultoftheneighbors.Alsoitdidjustreturntheneighbors_idvalue.
//OLD
[..]FORnINGRAPH_NEIGHBORS(@graphName,@startId)RETURNn
//NEW
[..]FORnINANY@startIdGRAPH@graphNameOPTIONS{bfs:true,uniqueVertices:'global'}RETURNn
OptionneighborExamples
SeeGRAPH_VERTICESonhowtotransformexamplestoAQLFILTER.Applythefilterontheneighborvariablen.
OptionedgeExamples
SeeGRAPH_VERTICESonhowtotransformexamplestoAQLFILTER.Applythefilterontheedgevariablee.
Howeverthisisabitmorecomplicatedasitinterfereswiththeglobaluniquenesscheck.ForedgeExamplesitissufficentwhenanyedgepointingtotheneighbormatchesthefilter.Using{uniqueVertices:'global'}firstpicksanyedgerandomly.Thanitchecksagainstthisedgeonly.Ifweknowtherearenovertexpairswithmultipleedgesbetweenthemwecanusethesimplevariantwhichissave:
//OLD
[..]FORnINGRAPH_NEIGHBORS(@graphName,@startId,{edgeExample:{label:'friend'}})RETURNe
//NEW
[..]FORn,eINANY@startIdGRAPH@graphNameOPTIONS{bfs:true,uniqueVertices:'global'}FILTERe.label=='friend'RETURN
n._id
Iftheremaybemultipleedgesbetweenthesamepairofverticeswehavetomakethedistinctcheckourselfesandcannotrelyonthetraverserdoingitcorrectlyforus:
//OLD
[..]FORnINGRAPH_NEIGHBORS(@graphName,@startId,{edgeExample:{label:'friend'}})RETURNe
//NEW
[..]FORn,eINANY@startIdGRAPH@graphNameOPTIONS{bfs:true}FILTERe.label=='friend'RETURNDISTINCTn._id
OptionvertexCollectionRestriction
IfweusethevertexCollectionRestrictionwehavetopostFiltertheneighborsbasedontheircollection.ThereforewecanmakeuseofthefunctionIS_SAME_COLLECTION:
//OLD
[..]FORnINGRAPH_NEIGHBORS(@graphName,@startId,{vertexCollectionRestriction:['vertices1','vertices2']})RETURNe
//NEW
[..]FORnINANY@startIdGRAPH@graphNameOPTIONS{bfs:true,uniqueVertices:true}FILTERIS_SAME_COLLECTION(vertices1,n)O
RIS_SAME_COLLECTION(vertices2,n)RETURNDISTINCTn._id
GRAPH_COMMON_NEIGHBORS
GRAPH_COMMON_NEIGHBORSisdefinedastwoGRAPH_NEIGHBORSqueriesandthanformingtheINTERSECTIONofbothqueries.HowtotranslatetheoptionspleaserefertoGRAPH_NEIGHBORS.Finallywehavetobuildtheoldresultformat{left,right,neighbors}.Ifyoujustneedpartsoftheresultyoucanadaptthisquerytoyourspecificneeds.
//OLD
Migratingnamedgraphfunctionsto3.0
35
FORvINGRAPH_COMMON_NEIGHBORS(@graphName,'vertices/1','vertices/2',{direction:'any'})RETURNv
//NEW
LETn1=(//Neighborsforvertex1Example
FORnINANY'vertices/1'GRAPH'graph'OPTIONS{bfs:true,uniqueVertices:"global"}RETURNn._id
)
LETn2=(//Neighborsforvertex2Example
FORnINANY'vertices/2'GRAPH'graph'OPTIONS{bfs:true,uniqueVertices:"global"}RETURNn._id
)
LETcommon=INTERSECTION(n1,n2)//Gettheintersection
RETURN{//Producetheoriginalresult
left:'vertices/1',
right:'vertices/2,
neighbors:common
}
NOTE:Ifyouareusingexamplesinsteadof_idsyouhavetoaddafiltertomakesurethattheleftisnotequaltotherightstartvertex.Togiveyouanexamplewithasinglevertexcollectionvertices,thereplacementwouldlooklikethis:
//OLD
FORvINGRAPH_COMMON_NEIGHBORS(@graphName,{name:"Alice"},{name:"Bob"})RETURNv
//NEW
FORleftINvertices
FILTERleft.name=="Alice"
LETn1=(FORnINANYleftGRAPH'graph'OPTIONS{bfs:true,uniqueVertices:"global"}RETURNn._id)
FORrightINvertices
FILTERright.name=="Bob"
FILTERright!=left//Makesureleftisnotidenticaltoright
LETn2=(FORnINANYrightGRAPH'graph'OPTIONS{bfs:true,uniqueVertices:"global"}RETURNn._id)
LETneighbors=INTERSECTION(n1,n2)
FILTERLENGTH(neighbors)>0//Onlypairswithsharedneighborsshouldbereturned
RETURN{left:left._id,right:right._id,neighbors:neighbors}
GRAPH_PATHS
Thisfunctioncomputesallpathsoftheentiregraph(withagivenminDepthandmaxDepth)asyoucanimaginethisfeatureisextremelyexpensiveandshouldneverbeused.HoweverpathscanagainbereplacedbyAQLtraversal.Assumeweonlyhaveonevertexcollectionverticesagain.
Nooptions
Bydefaultpathsoflength0to10arereturned.Andcirclesarenotfollowed.
//OLD
RETURNGRAPH_PATHS('graph')
//NEW
FORstartINvertices
FORv,e,pIN0..10OUTBOUNDstartGRAPH'graph'RETURN{source:start,destination:v,edges:p.edges,vertices:p.vertices}
followCycles
IfthisoptionissetwehavetomodifytheoptionsofthetraversalbymodifyingtheuniqueEdgesproperty:
//OLD
RETURNGRAPH_PATHS('graph',{followCycles:true})
//NEW
FORstartINvertices
FORv,e,pIN0..10OUTBOUNDstartGRAPH'graph'OPTIONS{uniqueEdges:'none'}RETURN{source:start,destination:v,edges:p
.edges,vertices:p.vertices}
GRAPH_COMMON_PROPERTIES
Migratingnamedgraphfunctionsto3.0
36
Thisfeatureinvolvesseveralfull-collectionscansandthereforeisextremelyexpensive.IfyoureallyneedityoucantransformitwiththehelpofATTRIBUTES,KEEPandZIP.
Startwithsingle_id
//OLD
RETURNGRAPH_COMMON_PROPERTIES('graph',"vertices/1","vertices/2")
//NEW
LETleft=DOCUMENT("vertices/1")//getonedocument
LETright=DOCUMENT("vertices/2")//gettheotherone
LETshared=(FORaINATTRIBUTES(left)//findallsharedattributes
FILTERleft[a]==right[a]
ORa=='_id'//alwaysinclude_id
RETURNa)
FILTERLENGTH(shared)>1//Returnthemonlyiftheyshareanattribute
RETURNZIP([left._id],[KEEP(right,shared)])//Buildtheresult
StartwithvertexExamples
Againweassumeweonlyhaveasinglecollectionvertices.Wehavetotransformtheexamplesintofilters.Iterateoververticestofindallleftdocuments.Foreachleftdocumentiterateoververticesagaintofindmatchingrightdocuments.Andreturnthesharedattributesasabove:
//OLD
RETURNGRAPH_COMMON_PROPERTIES('graph',{answer:42},{foo:"bar"})
//NEW
FORleftINvertices
FILTERleft.answer==42
LETcommons=(
FORrightINvertices
FILTERright.foo=="bar"
FILTERleft!=right
LETshared=(FORaINATTRIBUTES(left)
FILTERleft[a]==right[a]
ORa=='_id'
RETURNa)
FILTERLENGTH(shared)>1
RETURNKEEP(right,shared))
FILTERLENGTH(commons)>0
RETURNZIP([left._id],[commons])
GRAPH_SHORTEST_PATH
AshortestpathcomputationisnowdoneviathenewSHORTEST_PATHAQLstatement.
Nooptions
//OLD
FORpINGRAPH_SHORTEST_PATH(@graphName,@startId,@targetId,{direction:'outbound'})RETURNp
//NEW
LETp=(//RunoneshortestPath
FORv,eINOUTBOUNDSHORTEST_PATH@startIdTO@targetIdGRAPH@graphName
//Wereturnobjectswithvertex,edgeandweightforeachvertexonthepath
RETURN{vertex:v,edge:e,weight:(IS_NULL(e)?0:1)}
)
FILTERLENGTH(p)>0//Weonlywantshortestpathsthatactuallyexist
RETURN{//Werebuildtheoldformat
vertices:p[*].vertex,
edges:p[*FILTERCURRENT.e!=null].edge,
distance:SUM(p[*].weight)
}
OptionsweightanddefaultWeight
Migratingnamedgraphfunctionsto3.0
37
ThenewAQLSHORTEST_PATHofferstheoptionsweightAttributeanddefaultWeight.
//OLD
FORpINGRAPH_SHORTEST_PATH(@graphName,@startId,@targetId,{direction:'outbound',weight:"weight",defaultWeight:80})RE
TURNp
//NEW
LETp=(//RunoneshortestPath
FORv,eINOUTBOUNDSHORTEST_PATH@startIdTO@targetIdGRAPH@graphName
//Wereturnobjectswithvertex,edgeandweightforeachvertexonthepath
RETURN{vertex:v,edge:e,weight:(IS_NULL(e)?0:(IS_NUMBER(e.weight)?e.weight:80))}
)
FILTERLENGTH(p)>0//Weonlywantshortestpathsthatactuallyexist
RETURN{//Werebuildtheoldformat
vertices:p[*].vertex,
edges:p[*FILTERCURRENT.e!=null].edge,
distance:SUM(p[*].weight)//Wehavetorecomputethedistanceifweneedit
}
GRAPH_DISTANCE_TO
GraphdistancetoonlydiffersbytheresultformatfromGRAPH_SHORTEST_PATH.SowefollowthetransformationforGRAPH_SHORTEST_PATH,removesomeunnecessaryparts,andchangethereturnformat
//OLD
FORpINGRAPH_DISTANCE_TO(@graphName,@startId,@targetId,{direction:'outbound'})RETURNp
//NEW
LETp=(//RunoneshortestPath
FORv,eINOUTBOUNDSHORTEST_PATH@startIdTO@targetIdGRAPH@graphName
//DIFFERENCEweonlyreturntheweightforeachedgeonthepath
RETURNIS_NULL(e)?0:1}
)
FILTERLENGTH(p)>0//Weonlywantshortestpathsthatactuallyexist
RETURN{//Werebuildtheoldformat
startVertex:@startId,
vertex:@targetId,
distance:SUM(p[*].weight)
}
GRAPH_TRAVERSALandGRAPH_TRAVERSAL_TREE
ThesehavebeenremovedandshouldbereplacedbythenativeAQLtraversal.Therearemanypotentialsolutionsusingthenewsyntax,buttheylargelydependonwhatexactlyyouaretryingtoachieveandwouldgobeyondthescopeofthiscookbook.Hereisoneexamplehowtodothetransition,usingtheworldgraphasdata:
In2.8,itwaspossibletouseGRAPH_TRAVERSAL()togetherwithacustomvisitorfunctiontofindleafnodesinagraph.Leafnodesareverticesthathaveinboundedges,butnooutboundedges.Thevisitorfunctioncodelookedlikethis:
varaqlfunctions=require("org/arangodb/aql/functions");
aqlfunctions.register("myfunctions::leafNodeVisitor",function(config,result,vertex,path,connected){
if(connected&&connected.length===0){
returnvertex.name+"("+vertex.type+")";
}
});
AndtheAQLquerytomakeuseofit:
LETparams={
order:"preorder-expander",
visitor:"myfunctions::leafNodeVisitor",
visitorReturnsResults:true
}
FORresultINGRAPH_TRAVERSAL("worldCountry","worldVertices/world","inbound",params)
RETURNresult
Migratingnamedgraphfunctionsto3.0
38
TotraversethegraphstartingatvertexworldVertices/worldusingnativeAQLtraversalandanamedgraph,wecansimplydo:
FORvIN0..10INBOUND"worldVertices/world"GRAPH"worldCountry"
RETURNv
Itwillgiveusallvertexdocumentsincludingthestartvertex(becausetheminimumdepthissetto0).Themaximumdepthissetto10,whichisenoughtofollowalledgesandreachtheleafnodesinthisgraph.
Thequerycanbemodifiedtoreturnaformattedpathfromfirsttolastnode:
FORv,e,pIN0..10INBOUND"worldVertices/world"GRAPH"worldCountry"
RETURNCONCAT_SEPARATOR("->",p.vertices[*].name)
Theresultlookslikethis(shortened):
[
"World",
"World->Africa",
"World->Africa->Coted'Ivoire",
"World->Africa->Coted'Ivoire->Yamoussoukro",
"World->Africa->Angola",
"World->Africa->Angola->Luanda",
"World->Africa->Chad",
"World->Africa->Chad->N'Djamena",
...
]
Aswecansee,allpossiblepathsofvaryinglengthsarereturned.Wearenotreallyinterestedinthem,butwestillhavetodothetraversaltogofromWorldallthewaytotheleafnodes(e.g.Yamoussoukro).Todetermineifavertexisreallythelastonthepathinthesenseofbeingaleafnode,wecanuseanothertraversalofdepth1tocheckifthereisatleastoneoutgoingedge-whichmeansthevertexisnotaleafnode,otherwiseitis:
FORvIN0..10INBOUND"worldVertices/world"GRAPH"worldCountry"
FILTERLENGTH(FORvvININBOUNDvGRAPH"worldCountry"LIMIT1RETURN1)==0
RETURNCONCAT(v.name,"(",v.type,")")
Usingthecurrentvertexvasstartingpoint,thesecondtraversalisperformed.Itcanreturnearlyafteroneedgewasfollowed(LIMIT1),becausewedon'tneedtoknowtheexactcountanditisfasterthisway.Wealsodon'tneedtheactualvertex,sowecanjustRETURN1asdummyvalueasanoptimization.Thetraversal(whichisasub-query)willreturnanemptyarrayincaseofaleafnode,and[1]otherwise.Sinceweonlywanttheleafnodes,weFILTERoutallnon-emptyarraysandwhatisleftaretheleafnodesonly.TheattributesnameandtypeareformattedthewaytheywerelikeintheoriginalJavaScriptcode,butnowwithAQL.Thefinalresultisalistofallcapitals:
[
"Yamoussoukro(capital)",
"Luanda(capital)",
"N'Djamena(capital)",
"Algiers(capital)",
"Yaounde(capital)",
"Ouagadougou(capital)",
"Gaborone(capital)",
"Asmara(capital)",
"Cairo(capital)",
...
]
ThereisnodirectsubstitutefortheGRAPH_TRAVERSAL_TREE()function.Theadvantageofthisfunctionwasthatits(possiblyhighlynested)resultdatastructureinherentlyrepresentedthe"longest"possiblepathsonly.WithnativeAQLtraversal,allpathsfromminimumtomaximumtraversaldeptharereturned,includingthe"short"pathsaswell:
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"GRAPH"worldCountry"
RETURNCONCAT_SEPARATOR("<-",p.vertices[*]._key)
Migratingnamedgraphfunctionsto3.0
39
[
"continent-north-america<-country-antigua-and-barbuda",
"continent-north-america<-country-antigua-and-barbuda<-capital-saint-john-s",
"continent-north-america<-country-barbados",
"continent-north-america<-country-barbados<-capital-bridgetown",
"continent-north-america<-country-canada",
"continent-north-america<-country-canada<-capital-ottawa",
"continent-north-america<-country-bahamas",
"continent-north-america<-country-bahamas<-capital-nassau"
]
Asecondtraversalwithdepth=1canbeusedtocheckifwereachedaleafnode(nomoreincomingedges).Basedonthisinformation,the"short"pathscanbefilteredout.Notethatasecondconditionisrequired:itispossiblethatthelastnodeinatraversalisnotaleafnodeifthemaximumtraversaldepthisexceeded.Thus,weneedtoalsoletpathsthrough,whichcontainasmanyedgesashopswedointhetraversal(here:2).
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"GRAPH"worldCountry"
LETother=(
FORvv,eeININBOUNDvGRAPH"worldCountry"
//FILTERee!=e//needediftraversingedgesinANYdirection
LIMIT1
RETURN1
)
FILTERLENGTH(other)==0||LENGTH(p.edges)==2
RETURNCONCAT_SEPARATOR("<-",p.vertices[*]._key)
[
"continent-north-america<-country-antigua-and-barbuda<-capital-saint-john-s",
"continent-north-america<-country-barbados<-capital-bridgetown",
"continent-north-america<-country-canada<-capital-ottawa",
"continent-north-america<-country-bahamas<-capital-nassau"
]
Thefullpathscanbereturned,butitisnotinatree-likestructureaswithGRAPH_TRAVERSAL_TREE().Suchadatastructurecanbeconstructedonclient-sideifreallyneeded.
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"GRAPH"worldCountry"
LETother=(FORvv,eeININBOUNDvGRAPH"worldCountry"LIMIT1RETURN1)
FILTERLENGTH(other)==0||LENGTH(p.edges)==2
RETURNp
Pathdata(shortened):
[
{
"edges":[
{
"_id":"worldEdges/57585025",
"_from":"worldVertices/country-antigua-and-barbuda",
"_to":"worldVertices/continent-north-america",
"type":"is-in"
},
{
"_id":"worldEdges/57585231",
"_from":"worldVertices/capital-saint-john-s",
"_to":"worldVertices/country-antigua-and-barbuda",
"type":"is-in"
}
],
"vertices":[
{
"_id":"worldVertices/continent-north-america",
"name":"NorthAmerica",
"type":"continent"
},
{
"_id":"worldVertices/country-antigua-and-barbuda",
"code":"ATG",
Migratingnamedgraphfunctionsto3.0
40
"name":"AntiguaandBarbuda",
"type":"country"
},
{
"_id":"worldVertices/capital-saint-john-s",
"name":"SaintJohn's",
"type":"capital"
}
]
},
{
...
}
]
Thefirstandsecondvertexofthenthpathareconnectedbythefirstedge(p[n].vertices[0] p[n].edges[0]→p[n].vertices[1])andsoon.Thisstructuremightactuallybemoreconvenienttoprocesscomparedtoatree-likestructure.Notethattheedgedocumentsarealsoincluded,inconstrasttotheremovedgraphtraversalfunction.
Contactusviaoursocialchannelsifyouneedfurtherhelp.
Author:MichaelHackstein
Tags:#howto#aql#migration
Migratingnamedgraphfunctionsto3.0
41
MigratinganonymousgraphFunctionsfrom2.8orearlierto3.0
Problem
Withthereleaseof3.0allGRAPHfunctionshavebeendroppedfromAQLinfavorofamorenativeintegrationofgraphfeaturesintothequerylanguage.Ihaveusedtheoldgraphfunctionsandwanttoupgradeto3.0.
Graphfunctionscoveredinthisrecipe:
EDGESNEIGHBORSPATHSTRAVERSALTRAVERSAL_TREE
Solution
EDGES
TheEDGEScanbesimplyreplacedbyacalltotheAQLtraversal.
Nooptions
Thesyntaxisslightlydifferentbutmappingshouldbesimple:
//OLD
[..]FOReINEDGES(@@edgeCollection,@startId,'outbound')RETURNe
//NEW
[..]FORv,eINOUTBOUND@startId@@edgeCollectionRETURNe
UsingEdgeExamples
ExampleshavetobetransformedintoAQLfilterstatements.HowtodothispleasereadtheGRAPH_VERTICESsectioninMigratingGRAPH_*Functionsfrom2.8orearlierto3.0.Applythesefiltersontheedgevariablee.
OptionincluceVertices
Inordertoincludetheverticesyoujustusethevertexvariablevaswell:
//OLD
[..]FOReINEDGES(@@edgeCollection,@startId,'outbound',null,{includeVertices:true})RETURNe
//NEW
[..]FORv,eINOUTBOUND@startId@@edgeCollectionRETURN{edge:e,vertex:v}
NOTE:ThedirectioncannotbegivenasabindParameteranymoreithastobehard-codedinthequery.
NEIGHBORS
TheNEIGHBORSisabreadth-first-searchonthegraphwithaglobaluniquecheckforvertices.SowecanreplaceitbyaanAQLtraversalwiththeseoptions.Duetosyntaxchangesthevertexcollectionofthestartvertexisnolongermandatorytobegiven.YoumayhavetoadjustbindParameteresforthisquery.
Nooptions
Migratinganonymousgraphfunctionsto3.0
42
Thedefaultoptionsdidjustreturntheneighbors_idvalue.
//OLD
[..]FORnINNEIGHBORS(@@vertexCollection,@@edgeCollection,@startId,'outbound')RETURNn
//NEW
[..]FORnINOUTBOUND@startId@@edgeCollectionOPTIONS{bfs:true,uniqueVertices:'global'}RETURNn._id
NOTE:ThedirectioncannotbegivenasabindParameteranymoreithastobehard-codedinthequery.
UsingedgeExamples
ExampleshavetobetransformedintoAQLfilterstatements.HowtodothispleasereadtheGRAPH_VERTICESsectioninMigratingGRAPH_*Functionsfrom2.8orearlierto3.0.Applythesefiltersontheedgevariableewhichisthesecondreturnvariableofthetraversalstatement.
Howeverthisisabitmorecomplicatedasitinterfereswiththeglobaluniquenesscheck.ForedgeExamplesitissufficentwhenanyedgepointingtotheneighbormatchesthefilter.Using{uniqueVertices:'global'}firstpicksanyedgerandomly.Thanitchecksagainstthisedgeonly.Ifweknowtherearenovertexpairswithmultipleedgesbetweenthemwecanusethesimplevariantwhichissave:
//OLD
[..]FORnINNEIGHBORS(@@vertexCollection,@@edgeCollection,@startId,'outbound',{label:'friend'})RETURNn
//NEW
[..]FORn,eINOUTBOUND@startId@@edgeCollectionOPTIONS{bfs:true,uniqueVertices:'global'}
FILTERe.label=='friend'
RETURNn._id
Iftheremaybemultipleedgesbetweenthesamepairofverticeswehavetomakethedistinctcheckourselfesandcannotrelyonthetraverserdoingitcorrectlyforus:
//OLD
[..]FORnINNEIGHBORS(@@vertexCollection,@@edgeCollection,@startId,'outbound',{label:'friend'})RETURNn
//NEW
[..]FORn,eINOUTBOUND@startId@@edgeCollectionOPTIONS{bfs:true}
FILTERe.label=='friend'
RETURNDISTINCTn._id
OptionincludeData
Ifyouwanttoincludethedatasimplyreturnthecompletedocumentinsteadofonlythe_idvalue.
//OLD
[..]FORnINNEIGHBORS(@@vertexCollection,@@edgeCollection,@startId,'outbound',null,{includeData:true})RETURNn
//NEW
[..]FORn,eINOUTBOUND@startId@@edgeCollectionOPTIONS{bfs:true,uniqueVertices:'global'}RETURNn
PATHS
Thisfunctioncomputesallpathsoftheentireedgecollection(withagivenminDepthandmaxDepth)asyoucanimaginethisfeatureisextremelyexpensiveandshouldneverbeused.HoweverpathscanagainbereplacedbyAQLtraversal.
Nooptions
Bydefaultpathsoflength0to10arereturned.Andcirclesarenotfollowed.
//OLD
RETURNPATHS(@@vertexCollection,@@edgeCollection,"outbound")
//NEW
FORstartIN@@vertexCollection
Migratinganonymousgraphfunctionsto3.0
43
FORv,e,pIN0..10OUTBOUNDstart@@edgeCollectionRETURN{source:start,destination:v,edges:p.edges,vertices:p.vertice
s}
followCycles
IfthisoptionissetwehavetomodifytheoptionsofthetraversalbymodifyingtheuniqueEdgesproperty:
//OLD
RETURNPATHS(@@vertexCollection,@@edgeCollection,"outbound",{followCycles:true})
//NEW
FORstartIN@@vertexCollection
FORv,e,pIN0..10OUTBOUNDstart@@edgeCollectionOPTIONS{uniqueEdges:'none'}RETURN{source:start,destination:v,edges
:p.edges,vertices:p.vertices}
minDepthandmaxDepth
Ifthisoptionissetwehavetogivetheseparametersdirectlybeforethedirection.
//OLD
RETURNPATHS(@@vertexCollection,@@edgeCollection,"outbound",{minDepth:2,maxDepth:5})
//NEW
FORstartIN@@vertexCollection
FORv,e,pIN2..5OUTBOUNDstart@@edgeCollection
RETURN{source:start,destination:v,edges:p.edges,vertices:p.vertices}
TRAVERSALandTRAVERSAL_TREE
ThesehavebeenremovedandshouldbereplacedbythenativeAQLtraversal.Therearemanypotentialsolutionsusingthenewsyntax,buttheylargelydependonwhatexactlyyouaretryingtoachieveandwouldgobeyondthescopeofthiscookbook.Hereisoneexamplehowtodothetransition,usingtheworldgraphasdata:
In2.8,itwaspossibletouseTRAVERSAL()togetherwithacustomvisitorfunctiontofindleafnodesinagraph.Leafnodesareverticesthathaveinboundedges,butnooutboundedges.Thevisitorfunctioncodelookedlikethis:
varaqlfunctions=require("org/arangodb/aql/functions");
aqlfunctions.register("myfunctions::leafNodeVisitor",function(config,result,vertex,path,connected){
if(connected&&connected.length===0){
returnvertex.name+"("+vertex.type+")";
}
});
AndtheAQLquerytomakeuseofit:
LETparams={
order:"preorder-expander",
visitor:"myfunctions::leafNodeVisitor",
visitorReturnsResults:true
}
FORresultINTRAVERSAL(worldVertices,worldEdges,"worldVertices/world","inbound",params)
RETURNresult
TotraversethegraphstartingatvertexworldVertices/worldusingnativeAQLtraversalandananonymousgraph,wecansimplydo:
FORvIN0..10INBOUND"worldVertices/world"worldEdges
RETURNv
Itwillgiveusallvertexdocumentsincludingthestartvertex(becausetheminimumdepthissetto0).Themaximumdepthissetto10,whichisenoughtofollowalledgesandreachtheleafnodesinthisgraph.
Thequerycanbemodifiedtoreturnaformattedpathfromfirsttolastnode:
Migratinganonymousgraphfunctionsto3.0
44
FORv,e,pIN0..10INBOUND"worldVertices/world"e
RETURNCONCAT_SEPARATOR("->",p.vertices[*].name)
Theresultlookslikethis(shortened):
[
"World",
"World->Africa",
"World->Africa->Coted'Ivoire",
"World->Africa->Coted'Ivoire->Yamoussoukro",
"World->Africa->Angola",
"World->Africa->Angola->Luanda",
"World->Africa->Chad",
"World->Africa->Chad->N'Djamena",
...
]
Aswecansee,allpossiblepathsofvaryinglengthsarereturned.Wearenotreallyinterestedinthem,butwestillhavetodothetraversaltogofromWorldallthewaytotheleafnodes(e.g.Yamoussoukro).Todetermineifavertexisreallythelastonthepathinthesenseofbeingaleafnode,wecanuseanothertraversalofdepth1tocheckifthereisatleastoneoutgoingedge-whichmeansthevertexisnotaleafnode,otherwiseitis:
FORvIN0..10INBOUND"worldVertices/world"worldEdges
FILTERLENGTH(FORvvININBOUNDvworldEdgesLIMIT1RETURN1)==0
RETURNCONCAT(v.name,"(",v.type,")")
Usingthecurrentvertexvasstartingpoint,thesecondtraversalisperformed.Itcanreturnearlyafteroneedgewasfollowed(LIMIT1),becausewedon'tneedtoknowtheexactcountanditisfasterthisway.Wealsodon'tneedtheactualvertex,sowecanjustRETURN1asdummyvalueasanoptimization.Thetraversal(whichisasub-query)willreturnanemptyarrayincaseofaleafnode,and[1]otherwise.Sinceweonlywanttheleafnodes,weFILTERoutallnon-emptyarraysandwhatisleftaretheleafnodesonly.TheattributesnameandtypeareformattedthewaytheywerelikeintheoriginalJavaScriptcode,butnowwithAQL.Thefinalresultisalistofallcapitals:
[
"Yamoussoukro(capital)",
"Luanda(capital)",
"N'Djamena(capital)",
"Algiers(capital)",
"Yaounde(capital)",
"Ouagadougou(capital)",
"Gaborone(capital)",
"Asmara(capital)",
"Cairo(capital)",
...
]
ThereisnodirectsubstitutefortheTRAVERSAL_TREE()function.Theadvantageofthisfunctionwasthatits(possiblyhighlynested)resultdatastructureinherentlyrepresentedthe"longest"possiblepathsonly.WithnativeAQLtraversal,allpathsfromminimumtomaximumtraversaldeptharereturned,includingthe"short"pathsaswell:
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"worldEdges
RETURNCONCAT_SEPARATOR("<-",p.vertices[*]._key)
[
"continent-north-america<-country-antigua-and-barbuda",
"continent-north-america<-country-antigua-and-barbuda<-capital-saint-john-s",
"continent-north-america<-country-barbados",
"continent-north-america<-country-barbados<-capital-bridgetown",
"continent-north-america<-country-canada",
"continent-north-america<-country-canada<-capital-ottawa",
"continent-north-america<-country-bahamas",
"continent-north-america<-country-bahamas<-capital-nassau"
]
Migratinganonymousgraphfunctionsto3.0
45
Asecondtraversalwithdepth=1canbeusedtocheckifwereachedaleafnode(nomoreincomingedges).Basedonthisinformation,the"short"pathscanbefilteredout.Notethatasecondconditionisrequired:itispossiblethatthelastnodeinatraversalisnotaleafnodeifthemaximumtraversaldepthisexceeded.Thus,weneedtoalsoletpathsthrough,whichcontainasmanyedgesashopswedointhetraversal(here:2).
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"worldEdges
LETother=(
FORvv,eeININBOUNDvworldEdges
//FILTERee!=e//needediftraversingedgesinANYdirection
LIMIT1
RETURN1
)
FILTERLENGTH(other)==0||LENGTH(p.edges)==2
RETURNCONCAT_SEPARATOR("<-",p.vertices[*]._key)
[
"continent-north-america<-country-antigua-and-barbuda<-capital-saint-john-s",
"continent-north-america<-country-barbados<-capital-bridgetown",
"continent-north-america<-country-canada<-capital-ottawa",
"continent-north-america<-country-bahamas<-capital-nassau"
]
Thefullpathscanbereturned,butitisnotinatree-likestructureaswithTRAVERSAL_TREE().Suchadatastructurecanbeconstructedonclient-sideifreallyneeded.
FORv,e,pIN1..2INBOUND"worldVertices/continent-north-america"worldEdges
LETother=(FORvv,eeININBOUNDvworldEdgesLIMIT1RETURN1)
FILTERLENGTH(other)==0||LENGTH(p.edges)==2
RETURNp
Pathdata(shortened):
[
{
"edges":[
{
"_id":"worldEdges/57585025",
"_from":"worldVertices/country-antigua-and-barbuda",
"_to":"worldVertices/continent-north-america",
"type":"is-in"
},
{
"_id":"worldEdges/57585231",
"_from":"worldVertices/capital-saint-john-s",
"_to":"worldVertices/country-antigua-and-barbuda",
"type":"is-in"
}
],
"vertices":[
{
"_id":"worldVertices/continent-north-america",
"name":"NorthAmerica",
"type":"continent"
},
{
"_id":"worldVertices/country-antigua-and-barbuda",
"code":"ATG",
"name":"AntiguaandBarbuda",
"type":"country"
},
{
"_id":"worldVertices/capital-saint-john-s",
"name":"SaintJohn's",
"type":"capital"
}
]
},
{
...
Migratinganonymousgraphfunctionsto3.0
46
}
]
Thefirstandsecondvertexofthenthpathareconnectedbythefirstedge(p[n].vertices[0] p[n].edges[0]→p[n].vertices[1])andsoon.Thisstructuremightactuallybemoreconvenienttoprocesscomparedtoatree-likestructure.Notethattheedgedocumentsarealsoincluded,inconstrasttotheremovedgraphtraversalfunction.
Contactusviaoursocialchannelsifyouneedfurtherhelp.
Author:MichaelHackstein
Tags:#howto#aql#migration
Migratinganonymousgraphfunctionsto3.0
47
MigratingGRAPH_*Measurementsfrom2.8orearlierto3.0
Problem
Withthereleaseof3.0allGRAPHfunctionshavebeendroppedfromAQLinfavorofamorenativeintegrationofgraphfeaturesintothequerylanguage.Ihaveusedtheoldgraphfunctionsandwanttoupgradeto3.0.
Graphfunctionscoveredinthisrecipe:
GRAPH_ABSOLUTE_BETWEENNESSGRAPH_ABSOLUTE_CLOSENESSGRAPH_ABSOLUTE_ECCENTRICITYGRAPH_BETWEENNESSGRAPH_CLOSENESSGRAPH_DIAMETERGRAPH_ECCENTRICITYGRAPH_RADIUS
Solution1:UserDefinedFuntions
Registeringuser-definedfunctions
ThisstephastobeexecutedonceonArangoDBforeverydatabaseweareusing.
Weconnecttoarangodbwitharangoshtoissuethefollowingcommandstwo:
vargraphs=require("@arangodb/general-graph");
graphs._registerCompatibilityFunctions();
ThesehaveregisteredalloldGRAPH_*functionsasuser-definedfunctionsagain,withtheprefixarangodb::.
Modifytheapplicationcode
NextwehavetogothroughourapplicationcodeandreplaceallcallstoGRAPH_*byarangodb::GRAPH_*.Nowrunatestrunofourapplicationandcheckifitworked.Ifitworkedwearereadytogo.
ImportantInformation
Theuserdefinedfunctionswillcalltranslatedsubqueries(asdescribedinSolution2).Theoptimizerdoesnotknowanythingaboutthesesubqueriesbeforehandandcannotoptimizethewholeplan.Alsotheremightberead/writeconstellationsthatareforbiddeninuser-definedfunctions,thereforea"really"translatedquerymayworkwhiletheuser-definedfunctionworkaroundmayberejected.
Solution2:Foxx(recommended)Thegeneralgraphmodulestilloffersthemeasurementfunctions.AsthesearetypicallycomputationexpensiveandcreatelongrunningqueriesitisrecommendedtonotusethemincombinationwithotherAQLfeatures.ThereforethebestideaistoofferthesemeasurementsdirectlyviaanAPIusingFOXX.
FirstwecreateanewFoxxservice.Thenweincludethegeneral-graphmoduleintheservice.ForeverymeasurementweneedwesimplyofferaGETroutetoreadthismeasurement.
AsanexamplewedotheGRAPH_RADIUS:
///ADDFOXXCODEABOVE
constjoi=require('joi');
Migratinggraphmeasurementsto3.0
48
constcreateRouter=require('@arangodb/foxx/router');
constdd=require('dedent');
constrouter=createRouter();
constgraphs=require("@arangodb/general-graph");
router.get('/radius/:graph',function(req,res){
letgraph;
//Loadthegraph
try{
graph=graphs._graph(req.graph);
}catch(e){
res.throw('notfound');
}
res.json(graphs._radius());//Returntheradius
})
.pathParam('graph',joi.string().required(),'Thenameofthegraph')
.error('notfound','Graphwiththisnamedoesnotexist.')
.summary('ComputetheRadius')
.description(dd`
Thisfunctioncomputestheradiusofthegivengraph
andreturnsit.
`);
Author:MichaelHackstein
Tags:#howto#aql#migration
Migratinggraphmeasurementsto3.0
49
FulldepthGraph-Traversal
Problem
Letsassumeyouhaveadatabaseandsomeedgesandvertices.Nowyouneedthenodewiththemostconnectionsinfulldepth.
Solution
Youneedacustomtraversalwiththefollowingproperties:
StoreallverticesyouhavevisitedalreadyIfyouvisitanalreadyvisitedvertexreturntheconnections+1anddonottouchtheedgesIfyouvisitafreshvertexvisitallitschildrenandsumuptheirconnections.Storethissumandreturnit+1Repeatforallvertices.
vartraversal=require("org/arangodb/graph/traversal");
varknownFilter=function(config,vertex,path){
if(config.known[vertex._key]!==undefined){
return"prune";
}
return"";
};
varsumVisitor=function(config,result,vertex,path){
if(config.known[vertex._key]!==undefined){
result.sum+=config.known[vertex._key];
}else{
config.known[vertex._key]=result.sum;
}
result.sum+=1;
return;
};
varconfig={
datasource:traversal.collectionDatasourceFactory(db.e),//eismyedgecollection
strategy:"depthfirst",
order:"preorder",
filter:knownFilter,
expander:traversal.outboundExpander,
visitor:sumVisitor,
known:{}
};
vartraverser=newtraversal.Traverser(config);
varcursor=db.v.all();//vismyvertexcollection
while(cursor.hasNext()){
varnode=cursor.next();
traverser.traverse({sum:0},node);
}
config.known;//Returnstheresultoftypename:counter.Inarangoshthiswillprintoutcompleteresult
Toexecutethisscriptaccordinglyreplacedb.vanddb.ewithyourcollections(visvertices,eisedges)andwriteittoafile:(e.g.)traverse.jsthenexecuteitinarangosh:
cattraverse.js|arangosh
IfyouwanttouseitinproductionyoushouldhavealookattheFoxxframeworkwhichallowsyoutostoreandexecutethisscriptonserversideandmakeitaccessibleviayourownAPI:Foxx
Comment
FulldepthGraph-Traversal
51
Youonlycomputetheconnectionsofonevertexonceandcacheitthen.Complexityisalmostequaltotheamountofedges.Inthecodebelowconfig.knowncontainstheresultofallvertices,youthencanaddthesortingonit.
Author:MichaelHackstein
Tags:#graph
FulldepthGraph-Traversal
52
Usingacustomvisitorfromnode.js
Problem
Iwanttotraverseagraphusingacustomvisitorfromnode.js.
Solution
UsearangojsandanAQLquerywithacustomvisitor.
Installingarangojs
Firstthingistoinstallarangojs.Thiscanbedoneusingnpmorbower:
npminstallarangojs
or
bowerinstallarangojs
Exampledatasetup
Forthefollowingexample,weneedtheexamplegraphanddatafromhere.Pleasedownloadthecodefromthelinkandstoreitinthefilesystemusingafilenameofworld-graph-setup.js.ThenstarttheArangoShellandrunthecodefromthefile:
require("internal").load("/path/to/file/world-graph-setup.js");
Thescriptwillcreatethefollowingtwocollectionsandloadsomedataintothem:
v:acollectionwithvertexdocumentse:anedgecollectioncontainingtheconnectionsbetweenverticesinv
Registeringacustomvisitorfunction
Let'sregisteracustomvisitorfunctionnow.AcustomvisitorfunctionisaJavaScriptfunctionthatisexecutedeverytimethetraversalprocessesavertexinthegraph.
Toregisteracustomvisitorfunction,wecanexecutethefollowingcommandsintheArangoShell:
varaqlfunctions=require("org/arangodb/aql/functions");
aqlfunctions.register("myfunctions::leafNodeVisitor",function(config,result,vertex,path,connected){
if(connected&&connected.length===0){
returnvertex.name+"("+vertex.type+")";
}
});
Invokingthecustomvisitor
Thefollowingcodecanberuninnode.jstoexecuteanAQLquerythatwillmakeuseofthecustomvisitor:
Database=require('arangojs');
/*connectionthedatabase,changeasrequired*/
db=newDatabase('http://127.0.0.1:8529');
/*thequerystring*/
UsingacustomVisitor
53
varquery="FORresultINTRAVERSAL(v,e,@vertex,'inbound',@options)RETURNresult";
/*bindparameters*/
varbindVars={
vertex:"v/world",/*ourstartvertex*/
options:{
order:"preorder-expander",
visitor:"myfunctions::leafNodeVisitor",
visitorReturnsResults:true
}
};
db.query(query,bindVars,function(err,cursor){
if(err){
console.log('error:%j',err);
}else{
cursor.all(function(err2,list){
if(err){
console.log('error:%j',err2);
}else{
console.log("alldocumentkeys:%j",list);
}
});
}
});
Author:JanSteemann
Tags:#graph#traversal#aql#nodejs
UsingacustomVisitor
54
AQLExampleQueriesonanActorsandMoviesDatabase
Acknowledgments
OnStackoverflowtheuserVinczaskedforsomeexamplequeriesbasedongraphs.Socreditsforthisquestionsgotohim.Thedatasetsandquerieshavebeentakenfromtheguysofneo4j.Creditsandthankstothem.AsIalsothinkthisexamplesareyetmissingIdecidedtowritethisrecipe.
Problem
(CopyfromStackoverflow)Givenacollectionofactorsandacollectionofmovies.AndaactInedgescollection(withayearproperty)toconnectthevertex.
[Actor]←actin→[Movie]
HowcouldIget:
Allactorswhoactedin"movie1"OR"movie2"Allactorswhoactedinboth"movie1"AND"movie2"?Allcommonmoviesbetween"actor1"and"actor2"?Allactorswhoactedin3ormoremovies?Allmovieswhereexactly6actorsactedin?Thenumberofactorsbymovie?Thenumberofmoviesbyactor?Thenumberofmoviesactedinbetween2005and2010byactor?
Solution
Duringthissolutionwewillbeusingarangoshtocreateandquerythedata.AlltheAQLqueriesarestringsandcansimplybecopiedovertoyourfavoritedriverinsteadofarangosh.
CreateaTestDatasetinarangosh:
varactors=db._create("actors");
varmovies=db._create("movies");
varactsIn=db._createEdgeCollection("actsIn");
varTheMatrix=movies.save({_key:"TheMatrix",title:'TheMatrix',released:1999,tagline:'WelcometotheRealWorld'})._id;
varKeanu=actors.save({_key:"Keanu",name:'KeanuReeves',born:1964})._id;
varCarrie=actors.save({_key:"Carrie",name:'Carrie-AnneMoss',born:1967})._id;
varLaurence=actors.save({_key:"Laurence",name:'LaurenceFishburne',born:1961})._id;
varHugo=actors.save({_key:"Hugo",name:'HugoWeaving',born:1960})._id;
varEmil=actors.save({_key:"Emil",name:"EmilEifrem",born:1978});
actsIn.save(Keanu,TheMatrix,{roles:["Neo"],year:1999});
actsIn.save(Carrie,TheMatrix,{roles:["Trinity"],year:1999});
actsIn.save(Laurence,TheMatrix,{roles:["Morpheus"],year:1999});
actsIn.save(Hugo,TheMatrix,{roles:["AgentSmith"],year:1999});
actsIn.save(Emil,TheMatrix,{roles:["Emil"],year:1999});
varTheMatrixReloaded=movies.save({_key:"TheMatrixReloaded",title:"TheMatrixReloaded",released:2003,tagline:"Freeyo
urmind"});
actsIn.save(Keanu,TheMatrixReloaded,{roles:["Neo"],year:2003});
actsIn.save(Carrie,TheMatrixReloaded,{roles:["Trinity"],year:2003});
actsIn.save(Laurence,TheMatrixReloaded,{roles:["Morpheus"],year:2003});
actsIn.save(Hugo,TheMatrixReloaded,{roles:["AgentSmith"],year:2003});
varTheMatrixRevolutions=movies.save({_key:"TheMatrixRevolutions",title:"TheMatrixRevolutions",released:2003,tagline:
"Everythingthathasabeginninghasanend"});
actsIn.save(Keanu,TheMatrixRevolutions,{roles:["Neo"],year:2003});
actsIn.save(Carrie,TheMatrixRevolutions,{roles:["Trinity"],year:2003});
actsIn.save(Laurence,TheMatrixRevolutions,{roles:["Morpheus"],year:2003});
ExampleAQLQueriesforGraphs
55
actsIn.save(Hugo,TheMatrixRevolutions,{roles:["AgentSmith"],year:2003});
varTheDevilsAdvocate=movies.save({_key:"TheDevilsAdvocate",title:"TheDevil'sAdvocate",released:1997,tagline:'Evilhas
itswinningways'})._id;
varCharlize=actors.save({_key:"Charlize",name:'CharlizeTheron',born:1975})._id;
varAl=actors.save({_key:"Al",name:'AlPacino',born:1940})._id;
actsIn.save(Keanu,TheDevilsAdvocate,{roles:["KevinLomax"],year:1997});
actsIn.save(Charlize,TheDevilsAdvocate,{roles:["MaryAnnLomax"],year:1997});
actsIn.save(Al,TheDevilsAdvocate,{roles:["JohnMilton"],year:1997});
varAFewGoodMen=movies.save({_key:"AFewGoodMen",title:"AFewGoodMen",released:1992,tagline:"Intheheartofthenation'
scapital,inacourthouseoftheU.S.government,onemanwillstopatnothingtokeephishonor,andonewillstopatnothing
tofindthetruth."})._id;
varTomC=actors.save({_key:"TomC",name:'TomCruise',born:1962})._id;
varJackN=actors.save({_key:"JackN",name:'JackNicholson',born:1937})._id;
varDemiM=actors.save({_key:"DemiM",name:'DemiMoore',born:1962})._id;
varKevinB=actors.save({_key:"KevinB",name:'KevinBacon',born:1958})._id;
varKieferS=actors.save({_key:"KieferS",name:'KieferSutherland',born:1966})._id;
varNoahW=actors.save({_key:"NoahW",name:'NoahWyle',born:1971})._id;
varCubaG=actors.save({_key:"CubaG",name:'CubaGoodingJr.',born:1968})._id;
varKevinP=actors.save({_key:"KevinP",name:'KevinPollak',born:1957})._id;
varJTW=actors.save({_key:"JTW",name:'J.T.Walsh',born:1943})._id;
varJamesM=actors.save({_key:"JamesM",name:'JamesMarshall',born:1967})._id;
varChristopherG=actors.save({_key:"ChristopherG",name:'ChristopherGuest',born:1948})._id;
actsIn.save(TomC,AFewGoodMen,{roles:['Lt.DanielKaffee'],year:1992});
actsIn.save(JackN,AFewGoodMen,{roles:['Col.NathanR.Jessup'],year:1992});
actsIn.save(DemiM,AFewGoodMen,{roles:['Lt.Cdr.JoAnneGalloway'],year:1992});
actsIn.save(KevinB,AFewGoodMen,{roles:['Capt.JackRoss'],year:1992});
actsIn.save(KieferS,AFewGoodMen,{roles:['Lt.JonathanKendrick'],year:1992});
actsIn.save(NoahW,AFewGoodMen,{roles:['Cpl.JeffreyBarnes'],year:1992});
actsIn.save(CubaG,AFewGoodMen,{roles:['Cpl.CarlHammaker'],year:1992});
actsIn.save(KevinP,AFewGoodMen,{roles:['Lt.SamWeinberg'],year:1992});
actsIn.save(JTW,AFewGoodMen,{roles:['Lt.Col.MatthewAndrewMarkinson'],year:1992});
actsIn.save(JamesM,AFewGoodMen,{roles:['Pfc.LoudenDowney'],year:1992});
actsIn.save(ChristopherG,AFewGoodMen,{roles:['Dr.Stone'],year:1992});
varTopGun=movies.save({_key:"TopGun",title:"TopGun",released:1986,tagline:'Ifeeltheneed,theneedforspeed.'})._id;
varKellyM=actors.save({_key:"KellyM",name:'KellyMcGillis',born:1957})._id;
varValK=actors.save({_key:"ValK",name:'ValKilmer',born:1959})._id;
varAnthonyE=actors.save({_key:"AnthonyE",name:'AnthonyEdwards',born:1962})._id;
varTomS=actors.save({_key:"TomS",name:'TomSkerritt',born:1933})._id;
varMegR=actors.save({_key:"MegR",name:'MegRyan',born:1961})._id;
actsIn.save(TomC,TopGun,{roles:['Maverick'],year:1986});
actsIn.save(KellyM,TopGun,{roles:['Charlie'],year:1986});
actsIn.save(ValK,TopGun,{roles:['Iceman'],year:1986});
actsIn.save(AnthonyE,TopGun,{roles:['Goose'],year:1986});
actsIn.save(TomS,TopGun,{roles:['Viper'],year:1986});
actsIn.save(MegR,TopGun,{roles:['Carole'],year:1986});
varJerryMaguire=movies.save({_key:"JerryMaguire",title:'JerryMaguire',released:2000,tagline:'Therestofhislifebegins
now.'})._id;
varReneeZ=actors.save({_key:"ReneeZ",name:'ReneeZellweger',born:1969})._id;
varKellyP=actors.save({_key:"KellyP",name:'KellyPreston',born:1962})._id;
varJerryO=actors.save({_key:"JerryO",name:"JerryO'Connell",born:1974})._id;
varJayM=actors.save({_key:"JayM",name:'JayMohr',born:1970})._id;
varBonnieH=actors.save({_key:"BonnieH",name:'BonnieHunt',born:1961})._id;
varReginaK=actors.save({_key:"ReginaK",name:'ReginaKing',born:1971})._id;
varJonathanL=actors.save({_key:"JonathanL",name:'JonathanLipnicki',born:1996})._id;
actsIn.save(TomC,JerryMaguire,{roles:['JerryMaguire'],year:2000});
actsIn.save(CubaG,JerryMaguire,{roles:['RodTidwell'],year:2000});
actsIn.save(ReneeZ,JerryMaguire,{roles:['DorothyBoyd'],year:2000});
actsIn.save(KellyP,JerryMaguire,{roles:['AveryBishop'],year:2000});
actsIn.save(JerryO,JerryMaguire,{roles:['FrankCushman'],year:2000});
actsIn.save(JayM,JerryMaguire,{roles:['BobSugar'],year:2000});
actsIn.save(BonnieH,JerryMaguire,{roles:['LaurelBoyd'],year:2000});
actsIn.save(ReginaK,JerryMaguire,{roles:['MarceeTidwell'],year:2000});
actsIn.save(JonathanL,JerryMaguire,{roles:['RayBoyd'],year:2000});
varStandByMe=movies.save({_key:"StandByMe",title:"StandByMe",released:1986,tagline:"Forsome,it'sthelastrealtaste
ofinnocence,andthefirstrealtasteoflife.Butforeveryone,it'sthetimethatmemoriesaremadeof."})._id;
varRiverP=actors.save({_key:"RiverP",name:'RiverPhoenix',born:1970})._id;
varCoreyF=actors.save({_key:"CoreyF",name:'CoreyFeldman',born:1971})._id;
varWilW=actors.save({_key:"WilW",name:'WilWheaton',born:1972})._id;
varJohnC=actors.save({_key:"JohnC",name:'JohnCusack',born:1966})._id;
varMarshallB=actors.save({_key:"MarshallB",name:'MarshallBell',born:1942})._id;
actsIn.save(WilW,StandByMe,{roles:['GordieLachance'],year:1986});
ExampleAQLQueriesforGraphs
56
actsIn.save(RiverP,StandByMe,{roles:['ChrisChambers'],year:1986});
actsIn.save(JerryO,StandByMe,{roles:['VernTessio'],year:1986});
actsIn.save(CoreyF,StandByMe,{roles:['TeddyDuchamp'],year:1986});
actsIn.save(JohnC,StandByMe,{roles:['DennyLachance'],year:1986});
actsIn.save(KieferS,StandByMe,{roles:['AceMerrill'],year:1986});
actsIn.save(MarshallB,StandByMe,{roles:['Mr.Lachance'],year:1986});
varAsGoodAsItGets=movies.save({_key:"AsGoodAsItGets",title:'AsGoodasItGets',released:1997,tagline:'Acomedyfromthe
heartthatgoesforthethroat.'})._id;
varHelenH=actors.save({_key:"HelenH",name:'HelenHunt',born:1963})._id;
varGregK=actors.save({_key:"GregK",name:'GregKinnear',born:1963})._id;
actsIn.save(JackN,AsGoodAsItGets,{roles:['MelvinUdall'],year:1997});
actsIn.save(HelenH,AsGoodAsItGets,{roles:['CarolConnelly'],year:1997});
actsIn.save(GregK,AsGoodAsItGets,{roles:['SimonBishop'],year:1997});
actsIn.save(CubaG,AsGoodAsItGets,{roles:['FrankSachs'],year:1997});
varWhatDreamsMayCome=movies.save({_key:"WhatDreamsMayCome",title:'WhatDreamsMayCome',released:1998,tagline:'Afterlife
thereismore.Theendisjustthebeginning.'})._id;
varAnnabellaS=actors.save({_key:"AnnabellaS",name:'AnnabellaSciorra',born:1960})._id;
varMaxS=actors.save({_key:"MaxS",name:'MaxvonSydow',born:1929})._id;
varWernerH=actors.save({_key:"WernerH",name:'WernerHerzog',born:1942})._id;
varRobin=actors.save({_key:"Robin",name:'RobinWilliams',born:1951})._id;
actsIn.save(Robin,WhatDreamsMayCome,{roles:['ChrisNielsen'],year:1998});
actsIn.save(CubaG,WhatDreamsMayCome,{roles:['AlbertLewis'],year:1998});
actsIn.save(AnnabellaS,WhatDreamsMayCome,{roles:['AnnieCollins-Nielsen'],year:1998});
actsIn.save(MaxS,WhatDreamsMayCome,{roles:['TheTracker'],year:1998});
actsIn.save(WernerH,WhatDreamsMayCome,{roles:['TheFace'],year:1998});
varSnowFallingonCedars=movies.save({_key:"SnowFallingonCedars",title:'SnowFallingonCedars',released:1999,tagline:'Firs
tloveslast.Forever.'})._id;
varEthanH=actors.save({_key:"EthanH",name:'EthanHawke',born:1970})._id;
varRickY=actors.save({_key:"RickY",name:'RickYune',born:1971})._id;
varJamesC=actors.save({_key:"JamesC",name:'JamesCromwell',born:1940})._id;
actsIn.save(EthanH,SnowFallingonCedars,{roles:['IshmaelChambers'],year:1999});
actsIn.save(RickY,SnowFallingonCedars,{roles:['KazuoMiyamoto'],year:1999});
actsIn.save(MaxS,SnowFallingonCedars,{roles:['NelsGudmundsson'],year:1999});
actsIn.save(JamesC,SnowFallingonCedars,{roles:['JudgeFielding'],year:1999});
varYouveGotMail=movies.save({_key:"YouveGotMail",title:"You'veGotMail",released:1998,tagline:'Atoddsinlife...inlov
eon-line.'})._id;
varParkerP=actors.save({_key:"ParkerP",name:'ParkerPosey',born:1968})._id;
varDaveC=actors.save({_key:"DaveC",name:'DaveChappelle',born:1973})._id;
varSteveZ=actors.save({_key:"SteveZ",name:'SteveZahn',born:1967})._id;
varTomH=actors.save({_key:"TomH",name:'TomHanks',born:1956})._id;
actsIn.save(TomH,YouveGotMail,{roles:['JoeFox'],year:1998});
actsIn.save(MegR,YouveGotMail,{roles:['KathleenKelly'],year:1998});
actsIn.save(GregK,YouveGotMail,{roles:['FrankNavasky'],year:1998});
actsIn.save(ParkerP,YouveGotMail,{roles:['PatriciaEden'],year:1998});
actsIn.save(DaveC,YouveGotMail,{roles:['KevinJackson'],year:1998});
actsIn.save(SteveZ,YouveGotMail,{roles:['GeorgePappas'],year:1998});
varSleeplessInSeattle=movies.save({_key:"SleeplessInSeattle",title:'SleeplessinSeattle',released:1993,tagline:'Whatif
someoneyounevermet,someoneyouneversaw,someoneyouneverknewwastheonlysomeoneforyou?'})._id;
varRitaW=actors.save({_key:"RitaW",name:'RitaWilson',born:1956})._id;
varBillPull=actors.save({_key:"BillPull",name:'BillPullman',born:1953})._id;
varVictorG=actors.save({_key:"VictorG",name:'VictorGarber',born:1949})._id;
varRosieO=actors.save({_key:"RosieO",name:"RosieO'Donnell",born:1962})._id;
actsIn.save(TomH,SleeplessInSeattle,{roles:['SamBaldwin'],year:1993});
actsIn.save(MegR,SleeplessInSeattle,{roles:['AnnieReed'],year:1993});
actsIn.save(RitaW,SleeplessInSeattle,{roles:['Suzy'],year:1993});
actsIn.save(BillPull,SleeplessInSeattle,{roles:['Walter'],year:1993});
actsIn.save(VictorG,SleeplessInSeattle,{roles:['Greg'],year:1993});
actsIn.save(RosieO,SleeplessInSeattle,{roles:['Becky'],year:1993});
varJoeVersustheVolcano=movies.save({_key:"JoeVersustheVolcano",title:'JoeVersustheVolcano',released:1990,tagline:'Ast
oryoflove,lavaandburningdesire.'})._id;
varNathan=actors.save({_key:"Nathan",name:'NathanLane',born:1956})._id;
actsIn.save(TomH,JoeVersustheVolcano,{roles:['JoeBanks'],year:1990});
actsIn.save(MegR,JoeVersustheVolcano,{roles:['DeDe','AngelicaGraynamore','PatriciaGraynamore'],year:1990});
actsIn.save(Nathan,JoeVersustheVolcano,{roles:['Baw'],year:1990});
varWhenHarryMetSally=movies.save({_key:"WhenHarryMetSally",title:'WhenHarryMetSally',released:1998,tagline:'Atoddsin
life...inloveon-line.'})._id;
varBillyC=actors.save({_key:"BillyC",name:'BillyCrystal',born:1948})._id;
varCarrieF=actors.save({_key:"CarrieF",name:'CarrieFisher',born:1956})._id;
varBrunoK=actors.save({_key:"BrunoK",name:'BrunoKirby',born:1949})._id;
ExampleAQLQueriesforGraphs
57
actsIn.save(BillyC,WhenHarryMetSally,{roles:['HarryBurns'],year:1998});
actsIn.save(MegR,WhenHarryMetSally,{roles:['SallyAlbright'],year:1998});
actsIn.save(CarrieF,WhenHarryMetSally,{roles:['Marie'],year:1998});
actsIn.save(BrunoK,WhenHarryMetSally,{roles:['Jess'],year:1998});
Allactorswhoactedin"movie1"OR"movie2"Saywewanttofindallactorswhoactedin"TheMatrix"OR"TheDevilsAdvocate":
Firstletstrytogetallactorsforonemovie:
db._query("FORxINANY'movies/TheMatrix'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNx._id").toArray();
Result:
[
[
"actors/Keanu",
"actors/Hugo",
"actors/Emil",
"actors/Carrie",
"actors/Laurence"
]
]
NowwecontinuetoformaUNION_DISTINCToftwoNEIGHBORSquerieswhichwillbethesolution:
db._query("FORxINUNION_DISTINCT((FORyINANY'movies/TheMatrix'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETUR
Ny._id),(FORyINANY'movies/TheDevilsAdvocate'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNy._id))RETURNx"
).toArray();
[
[
"actors/Emil",
"actors/Hugo",
"actors/Carrie",
"actors/Laurence",
"actors/Keanu",
"actors/Al",
"actors/Charlize"
]
]
Allactorswhoactedinboth"movie1"AND"movie2"?Thisisalmostidenticaltothequestionabove.ButthistimewearenotintrestedinaUNIONbutinaINTERSECTION:
db._query("FORxININTERSECTION((FORyINANY'movies/TheMatrix'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURN
y._id),(FORyINANY'movies/TheDevilsAdvocate'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNy._id))RETURNx")
.toArray();
[
[
"actors/Keanu"
]
]
Allcommonmoviesbetween"actor1"and"actor2"?
ExampleAQLQueriesforGraphs
58
Thisisactuallyidenticaltothequestionaboutcommonactorsinmovie1andmovie2.Wejusthavetochangethestartingvertices.Asanexamplelet'sfindallmovieswhereHugoWeaving("Hugo")andKeanuReevesareco-starring:
db._query("FORxININTERSECTION((FORyINANY'actors/Hugo'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNy._id
),(FORyINANY'actors/Keanu'actsInOPTIONS{bfs:true,uniqueVertices:'global'}RETURNy._id))RETURNx").toArray();
[
[
"movies/TheMatrixRevolutions",
"movies/TheMatrixReloaded",
"movies/TheMatrix"
]
]
Allactorswhoactedin3ormoremovies?
Thisquestionisdifferent,wecannotmakeuseoftheneighborsfunctionhere.Insteadwewillmakeuseoftheedge-indexandtheCOLLECTstatementofAQLforgrouping.ThebasicideaistogroupalledgesbytheirstartVertex(whichinthisdatasetisalwaystheactor).Thenweremoveallactorswithlessthan3moviesfromtheresult.AsIamalsointerestedinthenumberofmoviesanactorhasactedin,Iincludedthevalueintheresultaswell:
db._query("FORxINactsInCOLLECTactor=x._fromWITHCOUNTINTOcounterFILTERcounter>=3RETURN{actor:actor,movies:co
unter}").toArray()
[
{
"actor":"actors/Carrie",
"movies":3
},
{
"actor":"actors/CubaG",
"movies":4
},
{
"actor":"actors/Hugo",
"movies":3
},
{
"actor":"actors/Keanu",
"movies":4
},
{
"actor":"actors/Laurence",
"movies":3
},
{
"actor":"actors/MegR",
"movies":5
},
{
"actor":"actors/TomC",
"movies":3
},
{
"actor":"actors/TomH",
"movies":3
}
]
Allmovieswhereexactly6actorsactedin?Thesameideaasinthequerybefore,butwithequalityfilter,howevernowweneedthemovieinsteadoftheactor,sowereturnthe_toattribute:
ExampleAQLQueriesforGraphs
59
db._query("FORxINactsInCOLLECTmovie=x._toWITHCOUNTINTOcounterFILTERcounter==6RETURNmovie").toArray()
[
"movies/SleeplessInSeattle",
"movies/TopGun",
"movies/YouveGotMail"
]
Thenumberofactorsbymovie?Werememberinourdataset_toontheedgecorrespondstothemovie,sowecounthowoftenthesame_toappears.Thisisthenumberofactors.ThequeryisalmostidenticaltotheonesbeforebutwithouttheFILTERafterCOLLECT:
db._query("FORxINactsInCOLLECTmovie=x._toWITHCOUNTINTOcounterRETURN{movie:movie,actors:counter}").toArray()
[
{
"movie":"movies/AFewGoodMen",
"actors":11
},
{
"movie":"movies/AsGoodAsItGets",
"actors":4
},
{
"movie":"movies/JerryMaguire",
"actors":9
},
{
"movie":"movies/JoeVersustheVolcano",
"actors":3
},
{
"movie":"movies/SleeplessInSeattle",
"actors":6
},
{
"movie":"movies/SnowFallingonCedars",
"actors":4
},
{
"movie":"movies/StandByMe",
"actors":7
},
{
"movie":"movies/TheDevilsAdvocate",
"actors":3
},
{
"movie":"movies/TheMatrix",
"actors":5
},
{
"movie":"movies/TheMatrixReloaded",
"actors":4
},
{
"movie":"movies/TheMatrixRevolutions",
"actors":4
},
{
"movie":"movies/TopGun",
"actors":6
},
{
"movie":"movies/WhatDreamsMayCome",
"actors":5
},
{
ExampleAQLQueriesforGraphs
60
"movie":"movies/WhenHarryMetSally",
"actors":4
},
{
"movie":"movies/YouveGotMail",
"actors":6
}
]
Thenumberofmoviesbyactor?Ithinkyougetthepicturebynow;)
db._query("FORxINactsInCOLLECTactor=x._fromWITHCOUNTINTOcounterRETURN{actor:actor,movies:counter}").toArray()
[
{
"actor":"actors/Al",
"movies":1
},
{
"actor":"actors/AnnabellaS",
"movies":1
},
{
"actor":"actors/AnthonyE",
"movies":1
},
{
"actor":"actors/BillPull",
"movies":1
},
{
"actor":"actors/BillyC",
"movies":1
},
{
"actor":"actors/BonnieH",
"movies":1
},
{
"actor":"actors/BrunoK",
"movies":1
},
{
"actor":"actors/Carrie",
"movies":3
},
{
"actor":"actors/CarrieF",
"movies":1
},
{
"actor":"actors/Charlize",
"movies":1
},
{
"actor":"actors/ChristopherG",
"movies":1
},
{
"actor":"actors/CoreyF",
"movies":1
},
{
"actor":"actors/CubaG",
"movies":4
},
{
"actor":"actors/DaveC",
"movies":1
ExampleAQLQueriesforGraphs
61
},
{
"actor":"actors/DemiM",
"movies":1
},
{
"actor":"actors/Emil",
"movies":1
},
{
"actor":"actors/EthanH",
"movies":1
},
{
"actor":"actors/GregK",
"movies":2
},
{
"actor":"actors/HelenH",
"movies":1
},
{
"actor":"actors/Hugo",
"movies":3
},
{
"actor":"actors/JackN",
"movies":2
},
{
"actor":"actors/JamesC",
"movies":1
},
{
"actor":"actors/JamesM",
"movies":1
},
{
"actor":"actors/JayM",
"movies":1
},
{
"actor":"actors/JerryO",
"movies":2
},
{
"actor":"actors/JohnC",
"movies":1
},
{
"actor":"actors/JonathanL",
"movies":1
},
{
"actor":"actors/JTW",
"movies":1
},
{
"actor":"actors/Keanu",
"movies":4
},
{
"actor":"actors/KellyM",
"movies":1
},
{
"actor":"actors/KellyP",
"movies":1
},
{
"actor":"actors/KevinB",
"movies":1
},
{
"actor":"actors/KevinP",
"movies":1
ExampleAQLQueriesforGraphs
62
},
{
"actor":"actors/KieferS",
"movies":2
},
{
"actor":"actors/Laurence",
"movies":3
},
{
"actor":"actors/MarshallB",
"movies":1
},
{
"actor":"actors/MaxS",
"movies":2
},
{
"actor":"actors/MegR",
"movies":5
},
{
"actor":"actors/Nathan",
"movies":1
},
{
"actor":"actors/NoahW",
"movies":1
},
{
"actor":"actors/ParkerP",
"movies":1
},
{
"actor":"actors/ReginaK",
"movies":1
},
{
"actor":"actors/ReneeZ",
"movies":1
},
{
"actor":"actors/RickY",
"movies":1
},
{
"actor":"actors/RitaW",
"movies":1
},
{
"actor":"actors/RiverP",
"movies":1
},
{
"actor":"actors/Robin",
"movies":1
},
{
"actor":"actors/RosieO",
"movies":1
},
{
"actor":"actors/SteveZ",
"movies":1
},
{
"actor":"actors/TomC",
"movies":3
},
{
"actor":"actors/TomH",
"movies":3
},
{
"actor":"actors/TomS",
"movies":1
ExampleAQLQueriesforGraphs
63
},
{
"actor":"actors/ValK",
"movies":1
},
{
"actor":"actors/VictorG",
"movies":1
},
{
"actor":"actors/WernerH",
"movies":1
},
{
"actor":"actors/WilW",
"movies":1
}
]
Thenumberofmoviesactedinbetween2005and2010byactor?ThisqueryiswhereaMultiModeldatabaseactuallyshines.Firstofallwewanttouseitinproduction,sowesetaskiplistindexonyear.Thisallowsastoexecutefastrangequerieslikebetween2005and2010.
db.actsIn.ensureSkiplist("year")
Nowweslightlymodifyourmoviesbyactorquery.Howevermydatasetcontainsonlyoldermovies,soIchangedtheyearrangefrom1990-1995:
db._query("FORxINactsInFILTERx.year>=1990&&x.year<=1995COLLECTactor=x._fromWITHCOUNTINTOcounterRETURN{acto
r:actor,movies:counter}").toArray()
[
{
"actor":"actors/BillPull",
"movies":1
},
{
"actor":"actors/ChristopherG",
"movies":1
},
{
"actor":"actors/CubaG",
"movies":1
},
{
"actor":"actors/DemiM",
"movies":1
},
{
"actor":"actors/JackN",
"movies":1
},
{
"actor":"actors/JamesM",
"movies":1
},
{
"actor":"actors/JTW",
"movies":1
},
{
"actor":"actors/KevinB",
"movies":1
},
{
"actor":"actors/KevinP",
"movies":1
},
ExampleAQLQueriesforGraphs
64
{
"actor":"actors/KieferS",
"movies":1
},
{
"actor":"actors/MegR",
"movies":2
},
{
"actor":"actors/Nathan",
"movies":1
},
{
"actor":"actors/NoahW",
"movies":1
},
{
"actor":"actors/RitaW",
"movies":1
},
{
"actor":"actors/RosieO",
"movies":1
},
{
"actor":"actors/TomC",
"movies":1
},
{
"actor":"actors/TomH",
"movies":2
},
{
"actor":"actors/VictorG",
"movies":1
}
]
CommentAuthor:MichaelHackstein
Tags:#graph#examples
ExampleAQLQueriesforGraphs
65
UseCases/ExamplesCrawlingGithubwithPromises
UsingArangoDBwithSails.js
PopulatingaTextbox
ExportingData
AccessingbasedocumentswithJava
AddXMLdatatoArangoDBwithJava
UseCases/Examples
66
CrawlingGithubwithPromises
Problem
ThenewArangoDBJavascriptdrivernolongerimposesanypromisesimplementation.Itfollowsthestandardcallbackpatternwithacallbackusingerrandres.
Butwhatifwewanttouseapromiselibrary-inthiscasethemostpopularonepromises?LetsgiveitatryandbuildagithubcrawlerwiththenewJavascriptdriverandpromises.
SolutionThefollowingsourcecodecanbefoundongithub.
PaginationwithPromisesmadeeasy
Thegithubdriverhasafunctiontogetallfollowers.However,theresultispaginated.Withtwohelperfunctionsandpromisesitisstraightforwardtoimplementafunctiontoretrieveallfollowersofanuser.
functionextractFollowers(name){
'usestrict';
returnnewPromise(function(resolve,reject){
github.user.getFollowers({user:name},promoteError(reject,function(res){
followPages(resolve,reject,[],res);
}));
});
}
ThefollowPagesfunctionsimplyextendstheresultwiththenextpageuntilthelastpageisreached.
functionfollowPages(resolve,reject,result,res){
'usestrict';
vari;
for(i=0;i<res.length;++i){
result.push(res[i]);
}
if(github.hasNextPage(res)){
github.getNextPage(res,promoteError(reject,function(res){
followPages(resolve,reject,result,res);
}));
}
else{
resolve(result);
}
}
Thepromoteerrorhelperisaconveniencefunctiontobridgecallbacksandpromises.
functionpromoteError(reject,resolve){
'usestrict';
returnfunction(err,res){
if(err){
if(err.hasOwnProperty("message")&&/ratelimitexceeded/.test(err.message)){
rateLimitExceeded=true;
}
console.error("caughterror:%s",err);
reject(err);
CrawlingGithubwithPromises
67
}
else{
resolve(res);
}
};
}
I'vedecidedtosticktothesequencereject(akaerr)followedbyresolve(akares)-likethecallbacks.ThepromoteErrorcanbeusedforthegithubcallbackaswellastheArangoDBdriver.
Queues,Queues,Queues
I'veonlyneededaverysimplejobqueue,soqueue-itisagoodchoice.ItprovidesaverysimpleAPIforhandlingjobqueues:
POST/queue/job
POST/queue/worker
DELETE/queue/job/:key
ThenewJavascriptdriverallowstoaccessarbitraryendpoint.FirstinstallaFoxximplementingthequeuemicroserviceinanArangoDBinstance.
foxx-managerinstallqueue-it/queue
Addinganewjobfromnode.jsisnoweasy
functionaddJob(data){
'usestrict';
returnnewPromise(function(resolve,reject){
db.endpoint("queue").post("job",data,promoteError(reject,resolve));
});
}
Transaction
Iwantedtocrawlusersandtheirrepos.Therelations("follows","owns","is_member","stars")isstoredinanedgecollection.Ionlyaddanedgeifitisnotalreadythere.ThereforeIcheckinsideatransaction,iftheedgeexistsandaddit,ifitdoesnot.
createRepoDummy(repo.full_name,data).then(function(dummyData){
returndb.transaction(
"relations",
String(function(params){
varme=params[0];
varyou=params[1];
vartype=params[2];
vardb=require("org/arangodb").db;
if(db.relations.firstExample({_from:me,_to:you,type:type})===null){
db.relations.save(me,you,{type:type});
}
}),
[meId,"repos/"+data._key,type],
function(err){
if(err){
throwerr;
}
returnhandleDummy(dummyData);
});
})
Pleasenotethattheactionfunctionisexecutedontheserverandnotinthenodejsclient.Thereforeweneedtopasstherelevantdataasparameters.Itisnotpossibletousetheclosurevariables.
RidingtheBeast
CrawlingGithubwithPromises
68
StartanArangoDBinstance(i.e.insideadockercontainer)andinstallthesimplequeue.
foxx-managerinstallqueue-it/queue
Startthearangoshandcreatecollectionsusers,reposandrelations.
arangosh>db._create("users");
arangosh>db.users.ensureHashIndex("name");
arangosh>db._create("repos");
arangosh>db.users.ensureHashIndex("name");
arangosh>db._createEdgeCollection("relations");
Noweverythingisinitialized.Fireupnodejsandstartcrawling:
node>varcrawler=require("./crawler");
node>crawler.github.authenticate({type:"basic",username:"username",password:"password"})
node>crawler.addJob({type:"user",identifier:"username"})
node>crawler.runJobs();
Comment
Pleasekeepinmindthatthisisjustanexperiment.Thereisnogooderrorhandlingandconveniencefunctionsforsetupandstart.Itisalsonotoptimizedforperformance.Forinstance,itwouldeasilybepossibletoavoidnodejs/ArangoDBroundtripsusingmoretransactions.
Sourcesusedinthisexample:
ArangoJSnpmpromisesArangoDBFoxxqueue-it
ThesourcecodeofthisexampleisavailablefromGithub:https://github.com/fceller/Foxxmender
Author:FrankCeller
Tags:#foxx#javascript#API#nodejs#driver
CrawlingGithubwithPromises
69
HowtouseArangoDBwithSails.jsFirstinstalltheSails.jsframeworkusingNPM:
npminstall-gsails
NowyoucancreateanewSails.jsappnamedsomenamewiththefollowingcommand:
sailsnewsomename
NowweneedtoaddArangoDBtothisnewapplication.Firstcdintothefreshlycreatedsomenamedirectory.TheninstalltheArangoDBadapterforSails.jswiththefollowingcommand:
npminstallsails-arangodb
Thishoweveronlyinstallsthenecessarydependency.Weneedtoconfiguretheapplicationtoloadtheadapter.Openthefileconfig/connections.js.Youwillseealistofexampleconfigurationsforpossibleconnectionstodatabases.Removetheonesyoudon'tneed(orjustkeepallofthem),andaddthefollowingconfiguration(adjustthehost,port,databasenameandgraphnametoyourneeds):
localArangoDB:{
adapter:'sails-arangodb',
host:'localhost',
port:8529,
database:{
name:'sails',
graph:'sails'
}
}
Now,youcanconfigureyourapptousetheArangoDBasyourdefaultconnection.Youdothisbyadjustingthefileconfig/models.js:
module.exports.models={
connection:'localArangoDB'//thisisthenamefromtheconnections.jsfile
//...
};
YourappisnowconfiguredtouseArangoDBforallmodelsbydefault.Youcannowforexamplecreateablueprintcontrollerbytypingthefollowinginyourconsole:
sailsgenerateapitodos
Nowyoucanbootyourapplicationwith:
sailslift
Youcannowaccesshttp://localhost:1337/todosandseeanemptylistoftodos.Andthencreateatodobyvisitinglocalhost:1337/todos/create?name=john.Thiswillcreatetheaccordingdocument(thathasanattributenamewiththevaluejohn)inthetodoscollectionoftheselecteddatabase.Youwillalsoseethedocumentwhenyouvisithttp://localhost:1337/todosagain.
Author:LucasDohmen
Tags:#nodejs
UsingArangoDBwithSails.js
70
Populatinganautocompletetextbox
Problem
Iwanttopopulateanautocompletetextboxwithvaluesfromacollection.Thecompletionsshouldadjustdynamicallybasedonuserinput.
Solution
Useawebframeworkfortheclient-sideautocompleterenderingandeventprocessing.Useacollectionwitha(sorted)skiplistindexandarangequeryonittoefficientlyfetchthecompletionvaluesdynamically.ConnectthetwousingasimpleFoxxroute.
Installanexampleapp
Thisappcontainsajquery-poweredwebpagewithanautocompletetextbox.Itusesjqueryautocomplete,buteveryotherwebframeworkwillalsodo.
Theappcanbeinstalledasfollows:
intheArangoDBwebinterface,switchintotheApplicationstabthere,clickAddApplicationswitchontheGithubtabforRepository,enterjsteemann/autocompleteforVersion,entermasterclickInstall
Nowenteramountpointfortheapplication.ThisistheURLpathunderwhichtheapplicationwillbecomeavailable.Fortheexampleapp,themountpointdoesnotmatter.ThewebpageintheexampleappassumesitisservedbyArangoDB,too.SoitusesarelativeURLautocomplete.Thisiseasiesttosetup,butinrealityyoumightwanttohaveyourwebpageservedbyadifferentserver.Inthiscase,yourwebpagewillhavetocalltheappmountpointyoujustentered.
Toseetheexampleappinaction,clickonOpen.Theautocompletetextboxshouldbepopulatedwithserverdatawhenatleasttwolettersareentered.
Backendcode,setupscript
Theappalsocontainsabackendroute/autocompletewhichiscalledbythewebpagetofetchcompletionsbasedonuserinput.TheHTMLcodeforthewebpageishere.
Containedintheappisasetupscriptthatwillcreateacollectionnamedcompletionsandloadsomeinitialdataintoit.TheexampleappprovidesautocompletionforUScitynames,andthesetupscriptpopulatesthecollectionwithabout10Kcitynames.
Thesetupscriptalsocreatesaskiplistindexonthelookupattribute,sothisattributecanbeusedforefficientfilteringandsortinglater.Thelookupattributecontainsthecitynamesalreadylower-cased,andtheoriginal(pretty)namesarestoredinattributepretty.Thisattributewillbereturnedtousers.
Backendcode,Foxxroutecontroller
Theappcontainsacontroller.Thebackendaction/autocompletethatiscalledbythewebpageisalsocontainedherein:
controller.get("/autocomplete",function(req,res){
//searchphraseenteredbyuser
varsearchString=req.params("q").trim()||"";
//lowerboundforsearchrange
varbegin=searchString.replace(/[^a-zA-Z]/g,"").toLowerCase();
if(begin.length===0){
//searchphraseisempty-noneedtoperfomasearchatall
res.json([]);
PopulatingaTextbox
71
return;
}
//upperboundforsearchrange
varend=begin.substr(0,begin.length-1)+String.fromCharCode(begin.charCodeAt(begin.length-1)+1);
//bindparametersforquery
varqueryParams={
"@collection":"completions",
"begin":begin,
"end":end
};
//thesearchquery
varquery="FORdocIN@@collectionFILTERdoc.lookup>=@begin&&doc.lookup<@endSORTdoc.lookupRETURN{label:doc.pre
tty,value:doc.pretty,id:doc._key}";
res.json(db._query(query,queryParams).toArray());
}
ThebackendcodefirstfetchesthesearchstringfromtheURLparameterq.Thisiswhatthewebpagewillsendus.
Basedonthesearchstring,alookuprangeiscalculated.Firstofall,thesearchstringislower-casedandallnon-lettercharactersareremovedfromit.Theresultingstringisthelowerboundforthelookup.Fortheupperbound,wecanusethelowerboundwithitslastlettercharactercodeincreasedbyone.
Forexample,iftheuserenteredLosAintothetextbox,thewebpagewillsendusthestringLosAinURLparameterq.Lower-casingandremovingnon-lettercharactersfromthestring,we'llgetlosa.Thisisthelowerbound.Theupperboundislosa,withitslastletteradjustedtob(i.e.losb).
Finally,thelowerandupperboundsareinsertedintothefollowingqueryusingbindparameters@beginand@end:
FORdocIN@@collection
FILTERdoc.lookup>=@begin&&doc.lookup<@end
SORTdoc.lookup
RETURN{
label:doc.pretty,
value:doc.pretty,
id:doc._key
}
Thecitynamesinthelookuprangewillbereturnedsorted.Foreachcity,threevaluesarereturned(theidcontainsthedocumentkey,theothertwovaluesarefordisplaypurposes).Otherframeworksmayrequireadifferentreturnformat,butthatcaneasilybedonebyadjustingtheAQLquery.
Author:JanSteemann
Tags:#aql#autocomplete#jquery
PopulatingaTextbox
72
ExportingDataforOfflineProcessingInthisrecipewewilllearnhowtousetheexportAPItoextractdataandprocessitwithPHP.AttheendoftherecipeyoucandownloadthecompletePHPscript.
Note:ThefollowingrecipeiswrittenusinganArangoDBserverwithversion2.6orhigher.Youcanalsousethedevelbranch,sinceversion2.6hasn'tbeenanofficialreleaseyet.
Howto
Importingexampledata
FirstofallweneedsomedatainanArangoDBcollection.Forthisexamplewewilluseacollectionnameduserswhichwewillpopulatewith100.000exampledocuments.ThiswayyoucangetthedataintoArangoDB:
#downloaddatafile
wgethttps://jsteemann.github.io/downloads/code/users-100000.json.tar.gz
#uncompressit
tarxvfzusers-100000.json.tar.gz
#importintoArangoDB
arangoimp--fileusers-100000.json--collectionusers--create-collectiontrue
SettingupArangoDB-PHP
ForthisrecipewewillusetheArangoDBPHPdriver:
gitclone-bdevel"https://github.com/arangodb/arangodb-php.git"
WewillnowwriteasimplePHPscriptthatestablishesaconnectiontoArangoDBonlocalhost:
<?php
namespacetriagens\ArangoDb;
//usethedriver'sautoloadertoloadclasses
require'arangodb-php/autoload.php';
Autoloader::init();
//setupconnectionoptions
$connectionOptions=array(
//endpointtoconnectto
ConnectionOptions::OPTION_ENDPOINT=>'tcp://localhost:8529',
//canuseKeep-Aliveconnection
ConnectionOptions::OPTION_CONNECTION=>'Keep-Alive',
//usebasicauthorization
ConnectionOptions::OPTION_AUTH_TYPE=>'Basic',
//userforbasicauthorization
ConnectionOptions::OPTION_AUTH_USER=>'root',
//passwordforbasicauthorization
ConnectionOptions::OPTION_AUTH_PASSWD=>'',
//timeoutinseconds
ConnectionOptions::OPTION_TIMEOUT=>30,
//databasename
ConnectionOptions::OPTION_DATABASE=>'_system'
);
try{
//establishconnection
$connection=newConnection($connectionOptions);
echo'Connected!'.PHP_EOL;
//TODO:nowdosomethingusefulwiththeconnection!
ExportingData
73
}catch(ConnectException$e){
print$e.PHP_EOL;
}catch(ServerException$e){
print$e.PHP_EOL;
}catch(ClientException$e){
print$e.PHP_EOL;
}
AfterrunningthescriptyoushouldseeConnected!inthebashifsuccessful.
Extractingthedata
Nowwecanrunanexportofthedatainthecollectionusers.PlacethefollowingcodeintotheTODOpartofthefirstcode:
functionexport($collection,Connection$connection){
$fp=fopen('output.json','w');
if(!$fp){
thrownewException('couldnotopenoutputfile!');
}
//settingstousefortheexport
$settings=array(
'batchSize'=>5000,//exportinchunksof5Kdocuments
'_flat'=>true//usesimplePHParrays
);
$export=newExport($connection,$collection,$settings);
//executetheexport.thiswillreturnanexportcursor
$cursor=$export->execute();
//statistics
$count=0;
$batches=0;
$bytes=0;
//nowwecanfetchthedocumentsfromthecollectioninbatches
while($docs=$cursor->getNextBatch()){
$output='';
foreach($docsas$doc){
$output.=json_encode($doc).PHP_EOL;
}
//writeoutchunk
fwrite($fp,$output);
//updatestatistics
$count+=count($docs);
$bytes+=strlen($output);
++$batches;
}
fclose($fp);
echosprintf('written%ddocumentsin%dbatcheswith%dtotalbytes',
$count,
$batches,
$bytes).PHP_EOL;
}
//runtheexport
export('users',$connection);
Thefunctionextractsalldocumentsfromthecollectionandwritesthemintoanoutputfileoutput.json.Inadditionitwillprintsomestatisticsaboutthenumberofdocumentsandthetotaldatasize:
written100000documentsin20batcheswith40890013totalbytes
Applyingsometransformations
ExportingData
74
WenowwillusePHPtotransformdataasweextractit:
functiontransformDate($value){
returnpreg_replace('/^(\\d+)-(\\d+)-(\\d+)$/','\\2/\\3/\\1',$value);
}
functiontransform(array$document){
static$genders=array('male'=>'m','female'=>'f');
$transformed=array(
'gender'=>$genders[$document['gender']],
'dob'=>transformDate($document['birthday']),
'memberSince'=>transformDate($document['memberSince']),
'fullName'=>$document['name']['first'].''.$document['name']['last'],
'email'=>$document['contact']['email'][0]
);
return$transformed;
}
functionexport($collection,Connection$connection){
$fp=fopen('output-transformed.json','w');
if(!$fp){
thrownewException('couldnotopenoutputfile!');
}
//settingstousefortheexport
$settings=array(
'batchSize'=>5000,//exportinchunksof5Kdocuments
'_flat'=>true//usesimplePHParrays
);
$export=newExport($connection,$collection,$settings);
//executetheexport.thiswillreturnanexportcursor
$cursor=$export->execute();
//nowwecanfetchthedocumentsfromthecollectioninbatches
while($docs=$cursor->getNextBatch()){
$output='';
foreach($docsas$doc){
$output.=json_encode(transform($doc)).PHP_EOL;
}
//writeoutchunk
fwrite($fp,$output);
}
fclose($fp);
}
//runtheexport
export('users',$connection);
Withthisscriptthefollowingchangeswillbemadeonthedata:
rewritethecontentsofthegenderattribute.femalebecomesfandmalebecomesmbirthdaynowbecomesdobthedateformationswillbechangedfromYYYY-MM-DDtoMM/DD/YYYYconcatenatethecontentsofname.firstandname.lastcontact.emailwillbetransformedfromanarraytoaflatstringeveryotherattributewillberemoved
Note:Theoutputwillbeinafilenamedoutput-transformed.json.
Filteringattributes
Excludecertainattributes
ExportingData
75
Insteadoffilteringoutasdoneinthepreviousexamplewecaneasilyconfiguretheexporttoexcludetheseattributesserver-side:
//settingstousefortheexport
$settings=array(
'batchSize'=>5000,//exportinchunksof5Kdocuments
'_flat'=>true,//usesimplePHParrays
'restrict'=>array(
'type'=>'exclude',
'fields'=>array('_id','_rev','_key','likes')
)
);
Thisscriptwillexcludetheattributes_id,_rev._keyandlikes.
Includecertainattributes
Wecanalsoincludeattributeswiththefollowingscript:
functionexport($collection,Connection$connection){
//settingstousefortheexport
$settings=array(
'batchSize'=>5000,//exportinchunksof5Kdocuments
'_flat'=>true,//usesimplePHParrays
'restrict'=>array(
'type'=>'include',
'fields'=>array('_key','name')
)
);
$export=newExport($connection,$collection,$settings);
//executetheexport.thiswillreturnanexportcursor
$cursor=$export->execute();
//nowwecanfetchthedocumentsfromthecollectioninbatches
while($docs=$cursor->getNextBatch()){
$output='';
foreach($docsas$doc){
$values=array(
$doc['_key'],
$doc['name']['first'].''.$doc['name']['last']
);
$output.='"'.implode('","',$values).'"'.PHP_EOL;
}
//printoutthedatadirectly
print$output;
}
}
//runtheexport
export('users',$connection);
Inthisscriptonlythe_keyandnameattributesareextracted.Intheprintsthe_key/namepairsareinCSVformat.
Note:Thewholescriptcanbedownloaded.
UsingtheAPIwithoutPHP
TheexportAPIRESTinterfacecanbeusedwithanyclientthatcanspeakHTTPlikecurl.Withthefollowingcommandyoucanfetchthedocumentsfromtheuserscollection:
curl
-XPOST
http://localhost:8529/_api/export?collection=users
--data'{"batchSize":5000}'
ExportingData
76
TheHTTPresponsewillcontatinaresultattributethatcontainstheactualdocuments.TheattributehasMorewillindicateiftherearemoredocumentsfortheclienttofetch.TheHTTPwillalsocontainanattributeidifsettotrue.
Withtheidyoucansendfollow-uprequestslikethis:
curl
-XPUT
http://localhost:8529/_api/export/13979338067709
Authors:ThomasSchmidtsandJanSteemann
Tags:#howto#php
ExportingData
77
HowtoretrievedocumentsfromArangoDBwithoutknowingthestructure?
Problem
IfyouuseaNoSQLdatabaseit'scommontoretrievedocumentswithanunknownattributestructure.Furthermore,theamountandtypesofattributesmaydifferindocumentsresultingfromasinglequery.Anotherproblemisthatyouwanttoaddoneoremoreattributestoadocument.
InJavayouareusedtoworkwithobjects.Regardingtheupperrequirementsitispossibletodirectlyretrieveobjectswiththesameattributestructureasthedocumentoutofthedatabase.Addingattributestoanobjectatruntimecouldbeverymessy.
Note:ArangoDB3.1andthecorrespondingJavadriverisneeded.
Solution
WiththelatestversionoftheJavadriverofArangoDBanobjectcalledBaseDocumentisprovided.
Thestructureisverysimple:Itonlyhasfourattributes:
publicclassBaseDocument{
Stringid;
Stringkey;
Stringrevision;
Map<String,Object>properties;
}
Thefirstthreeattributesarethesystemattributes_id,_keyand_rev.ThefourthattributeisaHashMap.ThekeyalwaysisaString,thevalueanobject.Thesepropertiescontainallnonsystemattributesofthedocument.
Themapcancontainvaluesofthefollowingtypes:
Map
ListBooleanNumberStringnull
Note:MapandListcontainobjects,whichareofthesametypesaslistedabove.
Toretrieveadocumentissimilartotheknownprocedure,exceptthatyouuseBaseDocumentastype.
ArangoDB.Builderarango=newArangoDB.Builder().builder();
DocumentEntity<BaseDocument>myObject=arango.db().collection("myCollection").getDocument("myDocumentKey",BaseDocument.class)
;
Otherresources
AccessingbasedocumentswithJava
78
MoredocumentationabouttheArangoDBJavadriverisavailable:
Tutorial:JavaintenminutesJavadriveratGithubExamplesourcecodeJavaDoc
Author:gschwab,MarkVollmary
Tags:#java#driver
AccessingbasedocumentswithJava
79
HowtoaddXMLdatatoArangoDB?
Problem
YouwanttostoreXMLdatafilesintoadatabasetohavetheabilitytomakequeriesontothem.
Note:ArangoDB3.1andthecorrespondingJavadriverisneeded.
SolutionSinceversion3.1.0thearagodb-java-driversupportswriting,readingandqueryingofrawstringscontainingtheJSONdocuments.
WithJsonMLyoucanconvertaXMLstringintoaJSONstringandbacktoXMLagain.
ConvertingXMLintoJSONwithJsonMLexample:
Stringstring="<recipename=\"bread\"prep_time=\"5mins\"cook_time=\"3hours\">"
+"<title>Basicbread</title>"
+"<ingredientamount=\"8\"unit=\"dL\">Flour</ingredient>"
+"<ingredientamount=\"10\"unit=\"grams\">Yeast</ingredient>"
+"<ingredientamount=\"4\"unit=\"dL\"state=\"warm\">Water</ingredient>"
+"<ingredientamount=\"1\"unit=\"teaspoon\">Salt</ingredient>"
+"<instructions>"
+"<step>Mixallingredientstogether.</step>"
+"<step>Kneadthoroughly.</step>"
+"<step>Coverwithacloth,andleaveforonehourinwarmroom.</step>"
+"<step>Kneadagain.</step>"
+"<step>Placeinabreadbakingtin.</step>"
+"<step>Coverwithacloth,andleaveforonehourinwarmroom.</step>"
+"<step>Bakeintheovenat180(degrees)Cfor30minutes.</step>"
+"</instructions>"
+"</recipe>";
JSONObjectjsonObject=JSONML.toJSONObject(string);
System.out.println(jsonObject.toString());
TheconvertedJSONstring:
{
"prep_time":"5mins",
"name":"bread",
"cook_time":"3hours",
"tagName":"recipe",
"childNodes":[
{
"childNodes":[
"Basicbread"
],
"tagName":"title"
},
{
"childNodes":[
"Flour"
],
"tagName":"ingredient",
"amount":8,
"unit":"dL"
},
{
"unit":"grams",
"amount":10,
"tagName":"ingredient",
"childNodes":[
"Yeast"
]
},
AddXMLdatatoArangoDBwithJava
80
{
"childNodes":[
"Water"
],
"tagName":"ingredient",
"amount":4,
"unit":"dL",
"state":"warm"
},
{
"childNodes":[
"Salt"
],
"tagName":"ingredient",
"unit":"teaspoon",
"amount":1
},
{
"childNodes":[
{
"tagName":"step",
"childNodes":[
"Mixallingredientstogether."
]
},
{
"tagName":"step",
"childNodes":[
"Kneadthoroughly."
]
},
{
"childNodes":[
"Coverwithacloth,andleaveforonehourinwarmroom."
],
"tagName":"step"
},
{
"tagName":"step",
"childNodes":[
"Kneadagain."
]
},
{
"childNodes":[
"Placeinabreadbakingtin."
],
"tagName":"step"
},
{
"tagName":"step",
"childNodes":[
"Coverwithacloth,andleaveforonehourinwarmroom."
]
},
{
"tagName":"step",
"childNodes":[
"Bakeintheovenat180(degrees)Cfor30minutes."
]
}
],
"tagName":"instructions"
}
]
}
SavingtheconvertedJSONtoArangoDBexample:
ArangoDB.Builderarango=newArangoDB.Builder().build();
ArangoCollectioncollection=arango.db().collection("testCollection")
DocumentCreateEntity<String>entity=collection.insertDocument(
jsonObject.toString());
Stringkey=entity.getKey();
AddXMLdatatoArangoDBwithJava
81
ReadingthestoredJSONasastringandconvertitbacktoXMLexample:
StringrawJsonString=collection.getDocument(key,String.class);
Stringxml=JSONML.toString(rawJsonString);
System.out.println(xml);
Exampleoutput:
<recipe_id="RawDocument/6834407522"_key="6834407522"_rev="6834407522"
cook_time="3hours"name="bread"prep_time="5mins">
<title>Basicbread</title>
<ingredientamount="8"unit="dL">Flour</ingredient>
<ingredientamount="10"unit="grams">Yeast</ingredient>
<ingredientamount="4"state="warm"unit="dL">Water</ingredient>
<ingredientamount="1"unit="teaspoon">Salt</ingredient>
<instructions>
<step>Mixallingredientstogether.</step>
<step>Kneadthoroughly.</step>
<step>Coverwithacloth,andleaveforonehourinwarmroom.</step>
<step>Kneadagain.</step>
<step>Placeinabreadbakingtin.</step>
<step>Coverwithacloth,andleaveforonehourinwarmroom.</step>
<step>Bakeintheovenat180(degrees)Cfor30minutes.</step>
</instructions>
</recipe>
Note:ThefieldsmandatorytoArangoDBdocumentsareadded;IftheybreakyourXMLschemayouhavetoremovethem.
Queryrawdataexample:
StringqueryString="FORtINtestCollectionFILTERt.cook_time=='3hours'RETURNt";
ArangoCursor<String>cursor=arango.db().query(queryString,null,null,String.class);
while(cursor.hasNext()){
JSONObjectjsonObject=newJSONObject(cursor.next());
Stringxml=JSONML.toString(jsonObject);
System.out.println("XMLvalue:"+xml);
}
OtherresourcesMoredocumentationabouttheArangoDBJavadriverisavailable:
Tutorial:JavaintenminutesJavadriveratGithubExamplesourcecodeJavaDoc
Author:AchimBrandt,MarkVollmary
Tags:#java#driver
AddXMLdatatoArangoDBwithJava
82
AdministrationUsingAuthentication
ImportingData
ReplicatingData
XCopyInstallWindows
Migrating2.8to3.0
Administration
83
Usingauthentication
Problem
IwanttouseauthenticationinArangoDB.
Solution
Inordertomakeauthenticationworkproperly,youwillneedtocreateuseraccountsfirst.
ThenadjustArangoDB'sconfigurationandturnonauthentication(ifit'soff).
Setuporadjustuseraccounts
ArangoDBuseraccountsarevalidthroughoutaserverinstanceanduserscanbegrantedaccesstooneormoredatabases.Theyaremanagedthroughthedatabasenamed_system.
Tomanageuseraccounts,connectwiththeArangoShelltotheArangoDBhostandthe_systemdatabase:
$arangosh--server.endpointtcp://127.0.0.1:8529--server.database"_system"
Bydefault,arangoshwillconnectwithausernamerootandanemptypassword.Thiswillworkifauthenticationisturnedoff.
Whenconnected,youcancreateanewuseraccountwiththefollowingcommand:
arangosh>require("org/arangodb/users").save("myuser","mypasswd");
myuserwillbetheusernameandmypasswdwillbetheuser'spassword.NotethatrunningthecommandlikethismaystorethepasswordliterallyinArangoShell'shistory.
Toavoidthat,useadynamicallycreatedpassword,e.g.:
arangosh>passwd=require("internal").genRandomAlphaNumbers(20);
arangosh>require("org/arangodb/users").save("myuser",passwd);
Theabovewillprintthepasswordonscreen(soyoucanmemorizeit)butwon'tstoreitinthecommandhistory.
Whilethere,youprobablywanttochangethepasswordofthedefaultrootusertoo.Otherwiseonewillbeabletoconnectwiththedefaultrootuseranditsemptypassword.Thefollowingcommandschangetherootuser'spassword:
arangosh>passwd=require("internal").genRandomAlphaNumbers(20);
arangosh>require("org/arangodb/users").update("root",passwd);
Turnonauthentication
AuthenticationisturnedonbydefaultinArangoDB.Youshouldmakesurethatitwasnotturnedoffmanuallyhowever.Checktheconfigurationfile(normallynamed/etc/arangodb.conf)andmakesureitcontainsthefollowinglineintheserversection:
authentication=true
ThiswillmakeArangoDBrequireauthenticationforeveryrequest(includingrequeststoFoxxapps).
IfyouwanttorunFoxxappswithoutHTTPauthentcation,butactivateHTTPauthenticationforthebuilt-inserverAPIs,youcanaddthefollowinglineintheserversectionoftheconfiguration:
authentication-system-only=true
UsingAuthentication
84
TheabovewillbypassauthenticationforrequeststoFoxxapps.
Whenfinishedmakingchanges,youneedtorestartArangoDB:
servicearangodbrestart
Checkaccessibility
Toconfirmauthenticationisineffect,tryconnectingtoArangoDBwiththeArangoShell:
$arangosh--server.endpointtcp://127.0.0.1:8529--server.database"_system"
Theabovewillimplicityuseausernamerootandanemptypasswordwhenconnecting.Ifyouchangedthepasswordoftherootaccountasdescribedabove,thisshouldnotworkanymore.
Youshouldalsovalidatethatyoucanconnectwithavaliduser:
$arangosh--server.endpointtcp://127.0.0.1:8529--server.database"_system"--server.usernamemyuser
YoucanalsousecurltocheckthatyouareactuallygettingHTTP401(Unauthorized)serverresponsesforrequeststhatrequireauthentication:
$curl--dump-http://127.0.0.1:8529/_api/version
Author:JanSteemann
Tags:#authentication#security
UsingAuthentication
85
Importingdata
Problem
IwanttoimportdatafromafileintoArangoDB.
Solution
ArangoDBcomeswithacommand-linetoolutilitynamedarangoimp.ThisutilitycanbeusedforimportingJSON-encoded,CSV,andtab-separatedfilesintoArangoDB.
arangoimpneedstobeinvokedfromthecommand-lineonceforeachimportfile.Thetargetcollectioncanalreadyexistorcanbecreatedbytheimportrun.
ImportingJSON-encodeddata
Inputformats
TherearetwosupportedinputformatsforimportingJSON-encodeddataintoArangoDB:
line-by-lineformat:ThisformatexpectseachlineintheinputfiletobeavalidJSONobjects.NolinebreaksmustoccurwithineachsingleJSONobject
arrayformat:ExpectsafilecontainingasinglearrayofJSONobjects.WhitespaceisallowedforformattinginsidetheJSONarrayandtheJSONobjects
Here'sanexamplefortheline-by-lineformatlookslikethis:
{"author":"FrankCeller","time":"2011-10-2608:42:49+0200","sha":"c413859392a45873936cbe40797970f8eed93ff9","message":"firstc
ommit","user":"f.celler"}
{"author":"FrankCeller","time":"2011-10-2621:32:36+0200","sha":"10bb77b8cc839201ff59a778f0c740994083c96e","message":"initial
release","user":"f.celler"}
...
Here'sanexampleforthesamedatainarrayformat:
[
{
"author":"FrankCeller",
"time":"2011-10-2608:42:49+0200",
"sha":"c413859392a45873936cbe40797970f8eed93ff9",
"message":"firstcommit",
"user":"f.celler"
},
{
"author":"FrankCeller",
"time":"2011-10-2621:32:36+0200",
"sha":"10bb77b8cc839201ff59a778f0c740994083c96e",
"message":"initialrelease",
"user":"f.celler"
},
...
]
ImportingJSONdatainline-by-lineformat
Anexampledatafileinline-by-lineformatcanbedownloadedhere.TheexamplefilecontainsallthecommitstotheArangoDBrepositoryasshownbygitlog--reverse.
Thefollowingcommandswillimportthedatafromthefileintoacollectionnamedcommits:
ImportingData
86
#downloadfile
wgethttp://jsteemann.github.io/downloads/code/git-commits-single-line.json
#actuallyimportdata
arangoimp--filegit-commits-single-line.json--collectioncommits--create-collectiontrue
Notethatnofiletypehasbeenspecifiedwhenarangoimpwasinvoked.Thisisbecausejsonisitsdefaultinputformat.
Theotherparametersusedhavethefollowingmeanings:
file:inputfilenamecollection:nameofthetargetcollectioncreate-collection:whetherornotthecollectionshouldbecreatedifitdoesnotexist
Theresultoftheimportprintedbyarangoimpshouldbe:
created:20039
warnings/errors:0
total:20039
Thecollectioncommitsshouldnowcontaintheexamplecommitdataaspresentintheinputfile.
ImportingJSONdatainarrayformat
Anexampleinputfileforthearrayformatcanbefoundhere.
ThecommandforimportingJSONdatainarrayformatissimilartowhatwe'vedonebefore:
#downloadfile
wgethttp://jsteemann.github.io/downloads/code/git-commits-array.json
#actuallyimportdata
arangoimp--filegit-commits-array.json--collectioncommits--create-collectiontrue
Thoughtheimportcommandisthesame(exceptthefilename),thereisanotabledifferencebetweenthetwoJSONformats:forthearrayformat,arangoimpwillreadandparsetheJSONinitsentiretybeforeitsendsanydatatotheArangoDBserver.Thatmeansthewholeinputfilemustfitintoarangoimp'sbuffer.Bydefault,arangoimpwillallocatea16MiBinternalbuffer,andinputfilesbiggerthanthatwillberejectedwiththefollowingmessage:
importfileistoobig.pleaseincreasethevalueof--batch-size(currently16777216).
SoforJSONinputfilesinarrayformatitmightbenecessarytoincreasethevalueof--batch-sizeinordertohavethefileimported.Alternatively,theinputfilecanbeconvertedtoline-by-lineformatmanually.
ImportingCSVdata
DatacanalsobeimportedfromaCSVfile.Anexamplefilecanbefoundhere.
The--typeparameterfortheimportcommandmustnowbesettocsv:
#downloadfile
wgethttp://jsteemann.github.io/downloads/code/git-commits.csv
#actuallyimportdata
arangoimp--filegit-commits.csv--typecsv--collectioncommits--create-collectiontrue
FortheCSVimport,thefirstlineintheinputfilehasaspecialmeaning:everyvaluelistedinthefirstlinewillbetreatedasanattributenameforthevaluesinallfollowinglines.Allfollowinglinesshouldalsohavethesamenumberof"columns".
"columns"insidetheCSVinputfilecanbeleftemptythough.Ifa"column"isleftemptyinaline,thenthisvaluewillbeomittedfortheimportsotherespectiveattributewillnotbesetintheimporteddocument.Notethatvaluesfromtheinputfilethatareenclosedindoublequoteswillalwaysbeimportedasstrings.Toimportnumericvalues,booleanvaluesorthenullvalue,don'tenclosethese
ImportingData
87
valuesinquotesintheinputfile.Notethatleadingzerosinnumericvalueswillberemoved.Importingnumberswithleadingzeroswillonlyworkwhenputtingthenumbersintostrings.
HereisanexampleCSVfile:
"author","time","sha","message"
"FrankCeller","2011-10-2608:42:49+0200","c413859392a45873936cbe40797970f8eed93ff9","firstcommit"
"FrankCeller","2011-10-2621:32:36+0200","10bb77b8cc839201ff59a778f0c740994083c96e","initialrelease"
...
arangoimpsupportsWindows(CRLF)andUnix(LF)linebreaks.Linebreaksmightalsooccurinsidevaluesthatareenclosedwiththequotecharacter.
ThedefaultseparatorforCSVfilesisthecomma.Itcanbechangedusingthe--separatorparameterwheninvokingarangoimp.Thequotecharacterdefaultstothedoublequote(").Tousealiteraldoublequoteinsidea"column"intheimportdata,usetwodoublequotes.Tochangethequotecharacter,usethe--quoteparameter.Touseabackslashforescapingquotecharacters,pleasesettheoption--backslash-escapetotrue.
Changingthedatabaseandserverendpoint
Bydefault,arangoimpwillconnecttothedefaultdatabaseon127.0.0.1:8529withausernamedroot.Tochangethis,usethefollowingparameters:
server.database:nameofthedatabasetousewhenimporting(default:_system)server.endpoint:addressoftheArangoDBserver(default:tcp://127.0.0.1:8529)
Usingauthentication
arangoimpwillbydefaultsendanusernamerootandanemptypasswordtotheArangoDBserver.ThisisArangoDB'sdefaultconfiguration,anditshouldbechanged.Tomakearangoimpuseadifferentusernameorpassword,thefollowingcommand-lineargumentscanbeused:
server.username:username,usedifauthenticationisenabledonserverserver.password:passwordforuser,usedifauthenticationisenabledonserver
Thepasswordargumentcanalsobeomittedinordertoavoidhavingitsavedintheshell'scommand-linehistory.Whenspecifyingausernamebutomittingthepasswordparameter,arangoimpwillpromptforapassword.
Additionalparameters
Bydefault,arangoimpwillimportdataintothespecifiedcollectionbutwillnottouchexistingdata.Oftenitisconvenienttofirstremovealldatafromacollectionandthenruntheimport.arangoimpsupportsthiswiththeoptional--overwriteflag.Whensettingittotrue,alldocumentsinthecollectionwillberemovedpriortotheimport.
Author:JanSteemann
Tags:#arangoimp#import
ImportingData
88
ReplicationThisSectionincludescookbookrecipesrelatedtotheReplicationtopic.
ReplicatingdatafromdifferentdatabasesSpeedingupslaveinitialization
Replication
89
Replicatingdatafromdifferentdatabases
Problem
Youhavetwoormoredifferentdatabaseswithvariousdatarespectivelycollectionsineachoneofthis,butyouwantyourdatatobecollectedatoneplace.
Note:ForthissolutionyouneedatleastArango2.0andyoumustrunthescriptineverydatabaseyouwanttobecollectdatafrom.
SolutionFirstofallyouhavetostartaserveronendpoint:
arangod--server.endpointtcp://127.0.0.1:8529
NowyouhavetocreatetwocollectionsandnamethemdataandreplicationStatus
db._create("data");
db._create("replicationStatus");
Savethefollowingscriptinafilenamedjs/common/modules/org/mysync.js
varinternal=require("internal");
//maximumnumberofchangesthatwecanhandle
varmaxChanges=1000;
//URLofcentralnode
vartransferUrl="http://127.0.0.1:8599/_api/import?collection=central&type=auto&createCollection=true&complete=true";
vartransferOptions={
method:"POST",
timeout:60
};
//thecollectionthatkeepsthestatusofwhatgotreplicatedtocentralnode
varreplicationCollection=internal.db.replicationStatus;
//thecollectioncontainingalldatachanges
varchangesCollection=internal.db.data;
functionkeyCompare(l,r){
if(l.length!=r.length){
returnl.length-r.length<0?-1:1;
}
//lengthisequal
for(i=0;i<l.length;++i){
if(l[i]!=r[i]){
returnl[i]<r[i]?-1:1;
}
}
return0;
};
functionlogger(msg){
"usestrict";
require("console").log("%s",msg);
}
functionreplicate(){
"usestrict";
ReplicatingData
90
varkey="status";//const
varstatus,newStatus;
try{
//fetchthepreviousreplicationstate
status=replicationCollection.document(key);
newStatus={_key:key,lastKey:status.lastKey};
}
catch(err){
//nopreviousreplicationstate.startfromthebeginning
newStatus={_key:key,lastKey:"0"};
}
//fetchthelatestchanges(needtoreversethembecause`last`returnsnewestchangesfirst)
varchanges=changesCollection.last(maxChanges).reverse(),change;
vartransfer=[];
for(changeinchanges){
if(changes.hasOwnProperty(change)){
vardoc=changes[change];
if(keyCompare(doc._key,newStatus.lastKey)<=0){
//alreadyhandledinapreviousreplicationrun
continue;
}
//documentsweneedtotransfer
//ifnecessary,wecouldrewritethedocumentshere,e.g.insert
//extravalues,createclient-specifickeysetc.
transfer.push(doc);
if(keyCompare(doc._key,newStatus.lastKey)>0){
//keeptrackofhighestkey
newStatus.lastKey=doc._key;
}
}
}
if(transfer.length===0){
//nothingtodo
logger("nothingtotransfer");
return;
}
logger("transferring"+transfer.length+"document(s)");
//nowtransferthedocumentstotheremoteserver
varresult=internal.download(transferUrl,JSON.stringify(transfer),transferOptions);
if(result.code>=200&&result.code<=202){
logger("centralserveracceptedthedocuments:"+JSON.stringify(result));
}
else{
//error
logger("centralserverdidnotacceptthedocuments:"+JSON.stringify(result));
throw"replicationerror";
}
//updatethereplicationstate
if(status){
//needtoupdatethepreviousreplicationstate
replicationCollection.update(key,newStatus);
}
else{
//needtoinsertthereplicationstate(1sttime)
replicationCollection.save(newStatus);
}
logger("deletingolddocuments");
//finallyremoveallelementsthatwetransferredsuccessfullyfromthechangescollection
//noneedtokeepthem
transfer.forEach(function(k){
changesCollection.remove(k);
});
}
exports.execute=function(param){
ReplicatingData
91
"usestrict";
logger("replicationwakeup");
replicate();
logger("replicationshutdown");
};
AfterwardschangetheURLofthecentralnodeinthescripttotheoneyouchosenbefore-e.g.tcp://127.0.0.1:8599
Nowregisterthescriptasarecurringaction:
require("internal").definePeriodic(1,10,"org/arangodb/mysync","execute","");
Note:Atthispointyoucanchangethetimethescriptwillbeexecuted.
Comment
Theserverstartedonendpointwillbethecentralnode.Itcollectschangesfromthelocalnodebyreplicatingitsdata.Thescriptwillpickupeverythingthathasbeenchangedsincethelastalterationinyourdatacollection.Every10seconds-orthetimeyouchosen-thescriptwillbeexecutedandsendthechangeddatatothecentralnodewhereitwillbeimportedintoacollectionnamedcentral.Afterthatthetransferreddatawillberemovedfromthedatacollection.
Ifyouwanttotestyourscriptsimplyaddsomedatatoyourdatacollection-e.g.:
for(i=0;i<100;++i)db.data.save({value:i});
Author:JanSteemann
Tags:#database#collection
ReplicatingData
92
Speedingupslaveinitialization
Problem
Youhaveaverybigdatabaseandwanttosetupamaster-slavereplicationbetweentwoormoreArangoDBinstances.Transferingtheentiredatabaseoverthenetworkmaytakealongtime,ifthedatabaseisbig.Inordertospeed-upthereplicationinitializationprocesstheslavecanbeinitializedusingabackupofthemaster.
Forthefollowingexamplesetup,wewillusetheinstancewithendpointtcp://master.domain.org:8529asmaster,andtheinstancewithendpointtcp://slave.domain.org:8530asslave.
Thegoalistohavealldatafromthedatabase_systemonmasterreplicatedtothedatabase_systemontheslave(thesameprocesscanbeappliedforotherdatabases).
Solution
Firstofallyouhavetostartthemasterserver,usingacommandliketheabove:
arangod--server.endpointtcp://master.domain.org:8529
Dependingonyourstorage-engineyoualsowanttoadjustthefollowingoptions:
ForMMFiles:
--wal.historic-logfiles(maximumnumberofhistoriclogfilestokeepaftercollection
(default:10))
ForRocksDB:
--rocksdb.wal-file-timeout(timeoutafterwhichunusedWALfilesaredeleted
inseconds(default:10))
TheoptionsabovepreventtheprematureremovalofoldWALfilesfromthemaster,andareusefulincaseintensewriteoperationshappenonthemasterwhileyouareinitializingtheslave.Infact,ifyoudonottunetheseoptions,whatcanhappenisthatthemasterWALfilesdonotincludeallthewriteoperationshappenedafterthebackupistaken.Thismayleadtosituationsinwhichtheinitializedslaveismissingsomedata,orfailstostart.
Nowyouhavetocreateadumpfromthemasterusingthetoolarangodump:
arangodump--output-directory"dump"--server.endpointtcp://master.domain.org:8529
Pleaseadaptthearangodumpcommandtoyourspecificcase.
Thefollowingisapossiblearangodumpoutput:
Serverversion:3.3
ConnectedtoArangoDB'tcp://master.domain.org:8529',database:'_system',username:'root'
Writingdumptooutputdirectory'dump'
Lasttickprovidedbyserveris:37276350
#Dumpingdocumentcollection'TestNums'...
#Dumpingdocumentcollection'TestNums2'...
#Dumpingdocumentcollection'frenchCity'...
#Dumpingdocumentcollection'germanCity'...
#Dumpingdocumentcollection'persons'...
#Dumpingedgecollection'frenchHighway'...
#Dumpingedgecollection'germanHighway'...
#Dumpingedgecollection'internationalHighway'...
#Dumpingedgecollection'knows'...
Processed9collection(s),wrote1298855504byte(s)intodatafiles,sent32batch(es)
SlaveInitialization
93
Inline4thelastservertickisdisplayed.Thisvaluewillbeusefulwhenwewillstartthereplication,tohavethereplication-applierstartreplicatingexactlyfromthattick.
Nextyouhavetostarttheslave:
arangod--server.endpointtcp://slave.domain.org:8530
Ifyouarerunningmasterandslaveonthesameserver(justfortest),pleasemakesureyougiveyourslaveadifferentdatadirectory.
Nowyouarereadytorestorethedumpwiththetoolarangorestore:
arangorestore--input-directory"dump"--server.endpointtcp://slave.domain.org:8530
Again,pleaseadaptthecommandaboveincaseyouareusingadatabasedifferentthan_system.
Oncetherestoreisfinishedtherearetwopossibleapproachestostartthereplication.
Approach1:All-in-onesetup
Startreplicationontheslavewitharangoshusingthefollowingcommand:
arangosh--server.endpointtcp://slave.domain.org:8530
db._useDatabase("_system");
require("@arangodb/replication").setupReplication({
endpoint:"tcp://master.domain.org:8529",
username:"myuser",
password:"mypasswd",
verbose:false,
includeSystem:false,
incremental:true,
autoResync:true
});
Thefollowingistheprintedoutput:
stillsynchronizing...lastreceivedstatus:2017-12-06T14:06:25Z:fetchingcollectionkeysforcollection'TestNums'from/_ap
i/replication/keys/keys?collection=7173693&to=57482456&serverId=24282855553110&batchId=57482462
stillsynchronizing...lastreceivedstatus:2017-12-06T14:06:25Z:fetchingcollectionkeysforcollection'TestNums'from/_ap
i/replication/keys/keys?collection=7173693&to=57482456&serverId=24282855553110&batchId=57482462
[...]
stillsynchronizing...lastreceivedstatus:2017-12-06T14:07:13Z:sorting10000000localkey(s)forcollection'TestNums'
stillsynchronizing...lastreceivedstatus:2017-12-06T14:07:13Z:sorting10000000localkey(s)forcollection'TestNums'
[...]
stillsynchronizing...lastreceivedstatus:2017-12-06T14:09:10Z:fetchingmastercollectiondumpforcollection'TestNums3',
type:document,id37276943,batch2,markersprocessed:15278,bytesreceived:2097258
stillsynchronizing...lastreceivedstatus:2017-12-06T14:09:18Z:fetchingmastercollectiondumpforcollection'TestNums5',
type:document,id37276973,batch5,markersprocessed:123387,bytesreceived:17039688
[...]
stillsynchronizing...lastreceivedstatus:2017-12-06T14:13:49Z:fetchingmastercollectiondumpforcollection'TestNums5',
type:document,id37276973,batch132,markersprocessed:9641823,bytesreceived:1348744116
stillsynchronizing...lastreceivedstatus:2017-12-06T14:13:59Z:fetchingcollectionkeysforcollection'frenchCity'from/_
api/replication/keys/keys?collection=27174045&to=57482456&serverId=24282855553110&batchId=57482462
{
"state":{
"running":true,
"lastAppliedContinuousTick":null,
"lastProcessedContinuousTick":null,
"lastAvailableContinuousTick":null,
"safeResumeTick":null,
"progress":{
"time":"2017-12-06T14:13:59Z",
"message":"sendbatchfinishcommandtourl/_api/replication/batch/57482462?serverId=24282855553110",
"failedConnects":0
},
"totalRequests":0,
"totalFailedConnects":0,
SlaveInitialization
94
"totalEvents":0,
"totalOperationsExcluded":0,
"lastError":{
"errorNum":0
},
"time":"2017-12-06T14:13:59Z"
},
"server":{
"version":"3.3.devel",
"serverId":"24282855553110"
},
"endpoint":"tcp://master.domain.org:8529",
"database":"_system"
}
Thisisthesamecommandthatyouwouldusetostartreplicationevenwithouttakingabackupfirst.Thedifference,inthiscase,isthatthedatathatispresentalreadyontheslave(andthathasbeenrestoredfromthebackup)thistimeisnottransferredoverthenetworkfromthemastertotheslave.
Thecommandabovewillonlycheckthatthedataalreadyincludedintheslaveisinsyncwiththemaster.Afterthischeck,thereplication-applierwillmakesurethatallwriteoperationsthathappenedonthemasterafterthebackuparereplicatedontheslave.
Whilethisapproachisdefinitelyfasterthantransferringthewholedatabaseoverthenetwork,sinceasynccheckisperformed,itcanstillrequiresometime.
Approach2:Applyreplicationbytick
Inthisapproach,thesynccheckdescribedaboveisnotperformed.Asaresultthisapproachisfasterastheexistingslavedataisnotchecked.Writeoperationsareexecutedstartingfromthetickyouprovideandcontinuewiththemaster'savailableticks.
Thisisstillasecurewaytostartreplicationasfarasthecorrecttickispassed.
Aspreviouslymentionedthelasttickprovidedbythemasterisdisplayedwhenusingarangodump.Inourexamplethelasttickwas37276350.
Firstofallyouhavetoapplythepropertiesofthereplication,usingarangoshontheslave:
arangosh--server.endpointtcp://slave.domain.org:8530
db._useDatabase("_system");
require("@arangodb/replication").applier.properties({
endpoint:"tcp://master.domain.org:8529",
username:"myuser",
password:"mypasswd",
verbose:false,
includeSystem:false,
incremental:true,
autoResync:true});
Thenyoucanstartthereplicationwiththelastprovidedlogtickofthemaster(outputofarangodump):
require("@arangodb/replication").applier.start(37276350)
Thefollowingistheprintedoutput:
{
"state":{
"running":true,
"lastAppliedContinuousTick":null,
"lastProcessedContinuousTick":null,
"lastAvailableContinuousTick":null,
"safeResumeTick":null,
"progress":{
"time":"2017-12-06T13:26:04Z",
"message":"applierinitiallycreatedfordatabase'_system'",
"failedConnects":0
},
SlaveInitialization
95
"totalRequests":0,
"totalFailedConnects":0,
"totalEvents":0,
"totalOperationsExcluded":0,
"lastError":{
"errorNum":0
},
"time":"2017-12-06T13:33:25Z"
},
"server":{
"version":"3.3.devel",
"serverId":"176090204017635"
},
"endpoint":"tcp://master.domain.org:8529",
"database":"_system"
}
Afterthereplicationhasbeenstartedwiththecommandabove,youcanusetheapplier.statecommandtocheckhowfarthelastappliedtickontheslaveisfarfromthelastavailablemastertick:
require("@arangodb/replication").applier.state()
{
"state":{
"running":true,
"lastAppliedContinuousTick":"42685113",
"lastProcessedContinuousTick":"42685113",
"lastAvailableContinuousTick":"57279944",
"safeResumeTick":"37276974",
"progress":{
"time":"2017-12-06T13:35:25Z",
"message":"fetchingmasterlogfromtick42685113,firstregulartick37276350,barrier:0,opentransactions:1",
"failedConnects":0
},
"totalRequests":190,
"totalFailedConnects":0,
"totalEvents":2704032,
"totalOperationsExcluded":0,
"lastError":{
"errorNum":0
},
"time":"2017-12-06T13:35:25Z"
},
"server":{
"version":"3.3.devel",
"serverId":"176090204017635"
},
"endpoint":"tcp://master.domain.org:8529",
"database":"_system"
}
Author:MaxKernbach
Tags:#database#replication#arangodump#arangorestore
SlaveInitialization
96
XCopyinstallArangoDBonWindows
Problem
Evenifthereisaniceguidedinstallerforwindowsusers,notalluserspreferthiskindofinstallation.InordertohaveaportableapplicationXCOPYdeploymentisnecessary.
Solution
AsofVersion2.5.1ArangoDBdoesn'trelyonregistryentriesanymoresowecandeployusingaZIP-file.
Steps
Unziparchive
Openanexplorer,chooseaplacewhereyouwantArangoDBtobeandunzipthefilesthere.Itwillcreateitsowntopleveldirectorywiththeversionnumberinthestring.
Alterconfiguration
Optional:
Editetc\arangodb3\arangod.confifthedefaultvaluesdon'tsuityourneedslike:
thelocationofthedatabasefilesportstobindstorageengine
andsoon.
CreateRuntimedirectories
arangodleansontheexistenceofsomedirectoriesinthevarsubdirectory,soyoushouldcreatethem:
C:\ProgramFiles\ArangoDB-3.1.11>mkdirvar\lib\arangodb
C:\ProgramFiles\ArangoDB-3.1.11>mkdirvar\lib\arangodb-apps
Runarangod
Tostartthedatabasesimplyrunit:
C:\ProgramFiles\ArangoDB-3.1.11>usr\bin\arangod
Nowittakesawhiletoopenallitsdatabases,loadsystemfacilities,bootstraptheJavaScriptenvironmentsandmanymore.Onceit'sreadytheoutputis:
INFOArangoDB(version3.1.11[windows])isreadyforbusiness.Havefun!
Nowyoucanopentheadministrativewebinterfaceinyourbrowserusinghttp://127.0.0.1:8529/.
Installingasservice
Ifyoudon'twanttorunarangodfromacmd-shelleachtimeinstallingitasasystemserviceistherightthingtodo.Thisrequiresadministrativeprivileges.YouneedtoRunasAdministratorthecmd-shell.FirstweneedtogranttheSYSTEM-useraccesstoourdatabasedirectory,sincearangodisgoingtoberunningasthatuser:
XCopyInstallWindows
97
C:\ProgramFiles\ArangoDB-3.1.11>icaclsvar/grantSYSTEM:F/t
Nextwecaninstalltheserviceitself:
C:\ProgramFiles\ArangoDB-3.1.11>usr\bin\arangod--install-service
NowyouwillhaveanewentryintheServicesdialoglabeledArangoDB-themulti-purposedatabase.Youcanstartitthereorjustdoitonthecommandlineusing:
C:\ProgramFiles\ArangoDB-3.1.11>NETSTARTArangoDB
Itwilltakeasimilaramountoftimetostartfromthecomandlineabovetilltheserviceisupandrunning.Sinceyoudon'thaveanyconsoletoinspectthestartup,messagesoftheseverityFATAL&ERRORarealsooutputintothewindowseventlog,soincaseoffailureyoucanhavealookattheEventlogintheManagementconsole
Author:WilfriedGoesgens
Tags:#windows#install
XCopyInstallWindows
98
InstallingArangoDBunattendedunderWindows
Problem
TheAvailableNSISbasedinstallerrequiresuserinteraction;Thismaybeunwantedforunattendedinstalli.e.viaChocolatey.
Solution
TheNSISinstallernowoffersa"SilentMode"whichallowsyoutorunitnoninteractiveandspecifyallchoicesavailableintheUIviacommandlineArguments.
TheoptionsareasallotherNSISoptionsspecifiedintheformof/OPTIONNAME=value.
SupportedoptionsForInstallation:
PASSWORD-Setthedatabasepassword.NewerversionswillalsotrytoevaluateaPASSWORDenvironmentvariable
INSTDIR-Installationdirectory.Adirectorywhereyouhaveaccessto.
DATABASEDIR-Databasedirectory.Adirectorywhereyouhaveaccesstoandthedatabasesshouldbecreated.APPDIR-FoxxServicesdirectory.Adirectorywhereyouhaveaccessto.INSTALL_SCOPE_ALL:
1-AllUsers+Service-launchthearangodbserviceviatheWindowsServices,installitforallusers0-SingleUser-installitintothehomeofthisuser,don'launchaservice.EventuallycreateadesktopIconsotheusercandothis.
DESKTOPICON-[0/1]whethertocreateIconsonthedesktoptoreferencearangoshandthewebinterfacePATH
0-don'talterthePATHenvironmentatall1:
INSTALL_SCOPE_ALL=1addittothepathforallusersINSTALL_SCOPE_ALL=0addittothepathofthecurrentlyloggedinusers
STORAGE_ENGINE-[auto/mmfiles/rocksdb]whichstorageenginetouse(arangodb3.2onwards)
ForUninstallation:
PURGE_DB-[0/1]ifsetto1thedatabasefilesArangoDBcreatedduringitslifetimewillberemovedtoo.
GenericOptionsderivedfromNSISS-silent-don'topentheUIduringinstallation
SilentNSISonWindows
99
MigrationfromArangoDB2.8to3.0
Problem
IwanttouseArangoDB3.0fromnowonbutIstillhavedatainArangoDB2.8.Ineedtomigratemydata.IamrunninganArangoDB3.0cluster(andpossiblyaclusterwithArangoDB2.8aswell).
Solution
TheinternaldataformatchangedcompletelyfromArangoDB2.8to3.0,thereforeyouhavetodumpalldatausingarangodumpandthenrestoreittothenewArangoDBinstanceusingarangorestore.
Generalinstructionsforthisprocedurecanbefoundinthemanual.Here,wecoversomeadditionaldetailsabouttheclustercase.
DumpingthedatainArangoDB2.8
Basically,dumpingthedataworkswiththefollowingcommand(usearangodumpfromyourArangoDB2.8distribution!):
arangodump--server.endpointtcp://localhost:8530--output-directorydump
oravariationofit,fordetailsseetheabovementionedmanualpageandthissection.IfyourArangoDB2.8instanceisacluster,simplyuseoneofthecoordinatorendpointsastheabove--server.endpoint.
RestoringthedatainArangoDB3.0
TheoutputconsistsofJSONfilesintheoutputdirectory,twoforeachcollection,oneforthestructureandoneforthedata.Thedataformatis100%compatiblewithArangoDB3.0,exceptthatArangoDB3.0hasanadditionaloptioninthestructurefilesforsynchronousreplication,namelytheattributereplicationFactor,whichisusedtospecify,howmanycopiesofthedataforeachshardarekeptinthecluster.
Therefore,youcansimplyusethiscommand(usethearangorestorefromyourArangoDB3.0distribution!):
arangorestore--server.endpointtcp://localhost:8530--input-directorydump
toimportyourdataintoyournewArangoDB3.0instance.Seethispagefordetailsontheavailablecommandlineoptions.IfyourArangoDB3.0instanceisacluster,thensimplyuseoneofthecoordinatorsas--server.endpoint.
Thatisit,yourdataismigrated.
Controlingthenumberofshardsandthereplicationfactor
Thisprocedureworksforallfourcombinationsofsingleserverandclusterforsourceanddestinationrespectively.Ifthetargetisasingleserverallsimplyworks.
Soitremainstoexplainhowonecontrolsthenumberofshardsandthereplicationfactorifthedestinationisacluster.
Ifthesourcewasacluster,arangorestorewillusethesamenumberofshardsasbefore,ifyoudonottellitotherwise.SinceArangoDB2.8doesnothavesynchronousreplication,itdoesnotproducedumpswiththereplicationFactorattribute,andsoarangorestorewillusereplicationfactor1forallcollections.Ifthesourcewasasingleserver,thesamewillhappen,additionally,arangorestorewillalwayscreatecollectionswithjustasingleshard.
Thereareessentially3waystochangethisbehaviour:
1. ThefirstistocreatethecollectionsexplicitlyontheArangoDB3.0cluster,andthensetthe--create-collectionfalseflag.Inthiscaseyoucancontrolthenumberofshardsandthereplicationfactorforeachcollectionindividuallywhenyoucreatethem.
2. Thesecondistousearangorestore'soptions--default-number-of-shardsand--default-replication-factor(thisoptionwas
Migrating2.8to3.0
100
introducedinVersion3.0.2)respectivelytospecifydefaultvalues,whicharetakenifthedumpfilesdonotspecifynumbers.Thismeansthatallsuchrestoredcollectionswillhavethesamenumberofshardsandreplicationfactor.
3. Ifyouneedmorecontrolyoucansimplyeditthestructurefilesinthedump.TheyaresimplyJSONfiles,youcanevenfirstuseaJSONprettyprintertomakeeditingeasier.ForthereplicationfactoryousimplyhavetoaddareplicationFactorattributetotheparameterssubobjectwithanumericalvalue.Forthenumberofshards,locatetheshardssubattributeoftheparametersattributeandeditit,suchthatithastherightnumberofattributes.Theactualnamesoftheattributesaswellastheirvaluesdonotmatter.Alternatively,addanumberOfShardsattributetotheparameterssubobject,thiswilloverridetheshardsattribute(thispossibilitywasintroducedinVersion3.0.2).
Notethatyoucanremoveindividualcollectionsfromyourdumpbydeletingtheirpairofstructureanddatafileinthedumpdirectory.Inthiswayyoucanrestoreyourdatainseveralstepsorevenparallelisetherestoreoperationbyrunningmultiplearangorestoreprocessesconcurrentlyondifferentdumpdirectories.Youshouldconsiderusingdifferentcoordinatorsforthedifferentarangorestoreprocessesinthiscase.
AllthesepossibilitiestogethergiveyoufullcontrolovertheshardinglayoutofyourdatainthenewArangoDB3.0cluster.
Migrating2.8to3.0
101
Showgrantsfunction
Problem
I'mlookingforuserdatabasegrants
Solution
Createaglobalfunctioninyour.arangosh.rcfilelikethis:
global.show_grants=function(){
letstmt;
stmt=db._createStatement({"query":"FORuin_usersRETURN{\"user\":u.user,\"databases\":u.databases}"});
console.log(stmt.execute().toString());
};
Nowwhenyouenterinarangosh,youcancallshow_grants()function.
Functionoutexample
[objectArangoQueryCursor,count:3,hasMore:false]
[
{
"user":"foo",
"databases":{
"_system":"rw",
"bar":"rw"
}
},
{
"user":"foo2",
"databases":{
"bar":"rw"
}
},
{
"user":"root",
"databases":{
"*":"rw"
}
}
]
Showgrantsfunction
102
CompilingArangoDB
Problem
YouwanttomodifysourcesoraddyourownchangestoArangoDB.
Solution
Arangodb,asmanyotheropensourceprojectsnowadaysisstandingontheshoulderofgiants.Thisgivesusasolidfoundationtobringyouauniqfeatureset,butitintroducesalotofdependenciesthatneedtobeinplaceinordertocompilearangodb.
SincebuildinfrastructuresareverydifferentdependingonthetargetOS,chooseyourtargetfromtherecepiesbelow.
CompileonDebian
CompileonWindows
RunningCustomBuild
Recompilingjemalloc
OpenSSL1.1
Compiling/Build
103
CompilingonDebian
Problem
Youwanttocompileandrunthedevelbranch,forexampletotestabugfix.InthisexamplethesystemisDebianbased.
Solution
ThissolutionwasmadeusingafreshDebianTestingmachineonAmazonEC2.Forcompleteness,thestepspertainingtoAWSarealsoincludedinthisrecipe.
LaunchtheVM
Optional
LogintoyourAWSaccountandlaunchaninstanceofDebianTesting.Iusedan'm3.xlarge'sincethathasabunchofcores,morethanenoughmemory,optimizednetworkandtheinstancestoreisonSSDswhichcanbeswitchedtoprovisionedIOPs.
TheCurrentAMIID'scanbefoundintheDebianWiki:https://wiki.debian.org/Cloud/AmazonEC2Image/Jessie
Upgradetotheverylatestversion
Optional
OnceyourEC2instanceisup,loginadadminandsudosutobecomeroot.
First,weremovethebackportsandchangetheprimarysources.list
rm-rf/etc/apt/sources.list.d
echo"debhttp://http.debian.net/debiantestingmaincontrib">/etc/apt/sources.list
echo"deb-srchttp://http.debian.net/debiantestingmaincontrib">>/etc/apt/sources.list
Updateandupgradethesystem.Makesureyoudon'thaveanybroken/unconfiguredpackages.Sometimesyouneedtorunsafe/fullupgrademorethanonce.Whenyou'redone,reboot.
apt-getinstallaptitude
aptitude-yupdate
aptitude-ysafe-upgrade
aptitude-yfull-upgrade
reboot
Installbuilddependencies
Mandatory
BeforeyoucanbuildArangoDB,youneedafewpackagespre-installedonyoursystem.
Loginagainandinstallthem.
sudoaptitude-yinstallgit-core\
build-essential\
libssl-dev\
libjemalloc-dev\
cmake\
python2.7\
sudoaptitude-yinstalllibldap2-dev#Enterpriseversiononly
DownloadtheSource
CompileonDebian
104
Downloadthelatestsourceusinggit:
unix>gitclonegit://github.com/arangodb/arangodb.git
Thiswillautomaticallyclonethedevelbranch.
Note:ifyouonlyplantocompileArangoDBlocallyanddonotwanttomodifyorpushanychanges,youcanspeedupcloningsubstantiallybyusingthe--single-branchand--depthparametersfortheclonecommandasfollows:
unix>gitclone--single-branch--depth1git://github.com/arangodb/arangodb.git
Setup
SwitchintotheArangoDBdirectory
unix>cdarangodb
unix>mkdirbuild
unix>cdbuild
Inordertogeneratethebuildenvironmentpleaseexecute
unix>cmake..
tosetuptheMakefiles.Thiswillcheckthevarioussystemcharacteristicsandinstalledlibraries.Ifyouinstalledthecompilerinanonstandardlocation,youmayneedtospecifyit:
cmake-DCMAKE_C_COMPILER=/opt/bin/gcc-DCMAKE_CXX_COMPILER=/opt/bin/g++..
IfyoucompileonMacOS,youshouldaddthefollowingoptionstothecmakecommand:
cmake..-DOPENSSL_ROOT_DIR=/usr/local/opt/openssl-DCMAKE_OSX_DEPLOYMENT_TARGET=10.11
IfyoualsoplantomakechangestothesourcecodeofArangoDB,youmaywanttocompilewiththeDebugbuildtype:
cmake..-DCMAKE_BUILD_TYPE=Debug
TheDebugtargetenablesadditionalsanitychecksetc.whichwouldslowdownproductionbinaries.Ifnobuildtypeisspecified,ArangoDBwillbecompiledwithbuildtypeRelWithDebInfo,whichisacompromisebetweengoodperformanceandmediumdebuggingexperience.
Otheroptionsvaluablefordevelopment:
-DUSE_MAINTAINER_MODE=On
NeededifyouplantomakechangestoAQLlanguage(whichisimplementedusingalexerandparserfilesinarangod/Aql/grammar.yandarangod/Aql/tokens.ll)orifyouwanttoenableruntimeassertions.Tousethemaintainermode,yoursystemhastocontainthetoolsFLEXandBISON.
-DUSE_BACKTRACE=On
UsethisoptionifyouwanttohaveC++stacktracesattachedtoyourexceptions.Thiscanbeusefultomorequicklylocatetheplacewhereanexceptionoranassertionwasthrown.Notethatthisoptionwillslowdowntheproducesbinariesabitandrequiresbuildingwithmaintainermode.
-DUSE_OPTIMIZE_FOR_ARCHITECTURE=On
CompileonDebian
105
Thiswilloptimizethebinaryforthetargetarchitecture,potentiallyenablingmorecompileroptimizations,butmakingtheresultingbinarylessportable.
ArangoDBwillthenautomaticallyusetheconfigurationfromfileetc/relative/arangod.conf.
-DUSE_FAILURE_TESTS=On
Thisoptionactivatesadditionalcodeintheserverthatintentionallymakestheservercrashormisbehave(e.g.bypretendingthesystemranoutofmemory)whencertaintestsarerun.Thisoptionisusefulforwritingtests.
-DUSE_JEMALLOC=Off
BydefaultArangoDBwillbebuiltwithabundledversionoftheJEMallocallocator.ThishoweverwillnotworkwhenusingruntimeanalyzerssuchasASANorValgrind.InordertousethesetoolsforinstrumentinganArangoDBbinary,JEMallocmustbeturnedoffduringcompilation.
sharedmemory
GypisusedasmakefilegeneratorbyV8.Gyprequiressharedmemorytobeavailable,whichmaynotifyoui.e.compileinachroot.Youcanmakeitavailablelikethis:
none/opt/chroots/ubuntu_precise_x64/dev/shmtmpfsrw,nosuid,nodev,noexec02
devpts/opt/chroots/ubuntu_precise_x64/dev/ptsdevptsgid=5,mode=62000
Compilation
Compiletheprograms(server,client,utilities)byexecuting
make
inthebuildsubdirectory.ThiswillcompileArangoDBandcreatethebinaryexecutableinfilebuild/bin/arangod.
Startingandtesting
Checkthebinarybystartingitusingthecommandline.
unix>build/bin/arangod-cetc/relative/arangod.conf--server.endpointtcp://127.0.0.1:8529/tmp/database-dir
ThiswillstartuptheArangoDBandlistenforHTTPrequestsonport8529boundtoIPaddress127.0.0.1.Youshouldseethestartupmessagessimilartothefollowing:
2016-06-01T12:47:29Z[29266]INFOArangoDBxxx...
2016-06-10T12:47:29Z[29266]INFOusingendpoint'tcp://127.0.0.1.8529'fornon-encryptedrequests
2016-06-01T12:47:30Z[29266]INFOAuthenticationisturnedon
2016-60-01T12:47:30Z[29266]INFOArangoDB(versionxxx)isreadyforbusiness.Havefun!
Ifitfailswithamessageaboutthedatabasedirectory,pleasemakesurethedatabasedirectoryyouspecifiedexistsandcanbewritteninto.
UseyourfavoritebrowsertoaccesstheURL
http://127.0.0.1:8529/
ThisshouldbringupArangoDB'swebinterface.
Re-buildingArangoDBafteranupdate
Tostayup-to-datewithchangesmadeinthemainArangoDBrepository,youwillneedtopullthechangesfromitandre-runmake.
CompileonDebian
106
Normally,thiswillbeassimpleasfollows:
unix>gitpull
unix>(cdbuild&&make)
FromtimetotimetherewillbebiggerstructuralchangesinArangoDB,whichmayrendertheoldMakefilesinvalid.Shouldthisbethecaseandmakecomplainsaboutmissingfilesetc.,thefollowingcommandsshouldfixit:
unix>rm-rfbuild/*
unix>cdbuild&&cmake..<cmakeoptionsgohere>
unix>(cdbuild&&make)
NotethattheabovecommandswillrunafullrebuildofArangoDBandallofitsthird-partycomponents.Thatwilltakeawhiletocomplete.
Installation
InalocaldevelopmentenvironmentitisnotnecessarytoinstallArangoDBsomewhere,becauseitcanbestartedfromwithinthesourcedirectoryasshownabove.
IfthereshouldbetheneedtoinstallArangoDB,executethefollowingcommand:
(cdbuild&&sudomakeinstall)
Theserverwillbydefaultbeinstalledin
/usr/local/sbin/arangod
Theconfigurationfilewillbeinstalledin
/usr/local/etc/arangodb/arangod.conf
Thedatabasewillbeinstalledin
/usr/local/var/lib/arangodb
TheArangoShellwillbeinstalledin
/usr/local/bin/arangosh
Youshouldaddanarangodbuserandgroup(asroot),plusmakesureitownsthesedirectories:
useradd-garangodbarangodb
chown-Rarangodb:arangodb/usr/local/var/lib/arangodb3-apps/
chown-Rarangodb:arangodb/tmp/database-dir/
Note:Theinstallationdirectorywillbedifferentifyouuseoneoftheprecompiledpackages.Pleasecheckthedefaultlocationsofyouroperatingsystem,e.g./etcand/var/lib.
WhenupgradingfromapreviousversionofArangoDB,pleasemakesureyouinspectArangoDB'slogfileafteranupgrade.ItmayalsobenecessarytostartArangoDBwiththe--database.auto-upgradeparameteroncetoperformrequiredupgradeorinitializationtasks.
Author:PatrickHuberAuthor:WilfriedGoesgens
Tags:#debian#driver
CompileonDebian
107
CompilingArangoDBunderWindows
Problem
IwanttocompileArangoDB3.0andonwardsunderWindows.
Note:ForthisrecipeyouneedatleastArangoDB3.0;ForArangoDBversionbefore3.0lookattheoldCompilingArangoDBunderWindows.
SolutionWithArangoDB3.0acompletecmakeenvironmentwasintroduced.Thisalsostreamlinesthedependenciesonwindows.Wesugesttousechocolatey.orgtoinstallmostofthedependencies.Forsuremostprojectsoffertheirownsetup&installpackages,chocolateyoffersasimplifiedwaytoinstallthemwithlessuserinteractions.Youcanevenusechocolateyviathebrandnewansibles2.0winrmfacilitytodounattendedinstallionsofsomesoftwareonwindows-thecoolthinglinuxguysalwaystoldyouabout.
Ingredients
Firstinstallthechocopackagemanagerbypastingthistinycmdletintoacommandwindow(needstoberunwithAdministratorprivileges;Rightclickstartmenu,CommandPrompt(Admin)):
@powershell-NoProfile-ExecutionPolicyBypass-Command"iex((new-objectnet.webclient).DownloadString('https://chocolatey.org
/install.ps1'))"&&SETPATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin
VisualStudioanditsCompiler
SincechococurrentlyfailstoaltertheenvironmentforMicrosoftVisualStudio,wesuggesttodownloadandinstallVisualStudiobyhand.CurrentlyVisualStudio2015istheonlysupportedoption.
Youneedtomakesurethatitinstallstheoption"ProgrammingLanguages/C++",elsecmakewillfailtodectectitlateron.
Afteritsuccessfullyinstalled,startitonce,soitcanfinishitssetup.
Moredependencies
Nowyoucaninvokethechocopackagemanagerforanunattendedinstallofthedependencies(needstoberunwithAdministratorprivilegesagain):
chocoinstall-ycmake.portablensispython2procdumpwindbgwgetnuget.commandline
ThenwefetchtheOpenSSLlibraryviathenugetcommandlineclient(doesn'tneedAdministratorprivileges):
nugetinstallopenssl
Optional
Ifyouintendtoruntheunittestsorcompilefromgit,youalsoneed(needstoberunwithAdministratorprivilegesagain):
chocoinstall-ygitwinflexbisonruby
CloseandreopentheAdministratorcommandwindowinordertocontinuewiththerubydevkit:
chocoinstall-yruby2.devkit
CompileonWindows
108
AndmanuallyinstalltherequirementsviatheGemfilefetchedfromtheArangoDBGitrepository(needstoberunwithAdministratorprivileges):
wgethttps://raw.githubusercontent.com/arangodb/arangodb/devel/UnitTests/HttpInterface/Gemfile
setPATH=%PATH%;C:\tools\DevKit2\bin;C:\tools\DevKit2\mingw\bin
geminstallbundler
bundler
NotethattheV8buildscriptsandgyparen'tcompatiblewithPython3.xhenceyouneedpython2!
BuildingArangoDB
Downloadandextractthereleasetarballfromhttps://www.arangodb.com/download/
Orclonethegithubrepository,andcheckoutthebranchortagyouneed(devel,3.0)
gitclonehttps://github.com/arangodb/arangodb.git-bdevel
cdarangodb
GeneratetheVisualstudioprojectfiles,andcheckbackthatcmakediscoveredallcomponentsonyoursystem:
mkdirBuild64
cdBuild64
cmake-G"VisualStudio14Win64"..
Notethatinsomecasescmakestrugglestofindtheproperpythoninterpreter(i.e.thecygwinonewon'twork).Youcanforceoverruleitbyappending:
-DPYTHON_EXECUTABLE:FILEPATH=C:/tools/python2/python.exe
YoucannowloadtheseintheVisualStudioIDEorusecmaketostartthebuild:
cmake--build.--configRelWithDebInfo
ThebinariesneedtheICUdatafileicudt54l.dat,whichisautomaticallycopiedintothedirectorycontainingtheexecutable.
Fordevelopment,unittestsanddocumentation:Cygwin(Optional)Thedocumentationandunittestsstillrequireacygwinenvironment.Herethehintshowtogetitproperlyinstalled:
Youneedatleastmakefromcygwin.Cygwinalsooffersacmake.Donotinstallthecygwincmake.
Youshouldalsoissuethesecommandstogenerateuserinformationsforthecygwincommands:
mkpasswd>/etc/passwd
mkgroup>/etc/group
TurningACLoff(noacl)forallmountsincygwinfixespermissionstroublesthatmayappearinthebuild:
#/etc/fstab
#
#ThisfileisreadoncebythefirstprocessinaCygwinprocesstree.
#Topickupchanges,restartallCygwinprocesses.Foradescription
#seehttps://cygwin.com/cygwin-ug-net/using.html#mount-table
#noacl=IgnoreAccessControlListandletWindowshandlepermissions
C:/cygwin64/bin/usr/binntfsbinary,auto,noacl00
C:/cygwin64/lib/usr/libntfsbinary,auto,noacl00
C:/cygwin64/ntfsoverride,binary,auto,noacl00
CompileonWindows
109
none/cygdrivecygdrivebinary,posix=0,user,noacl00
EnablenativesymlinksforCygwinandgitCygwinwillcreateproprietaryfilesasplaceholdersbydefaultinsteadofactuallysymlinkingfiles.TheplaceholderslatertellCygwinwheretoresolvepathsto.Itdoesnotintercepteveryaccesstotheplaceholdershowever,sothat3rdpartyscriptsbreak.WindowsVistaandabovesupportrealsymlinks,andCygwincanbeconfiguredtomakeuseofit:
#useactualsymlinkstopreventdocumentationbuilderrors
#(requireselevatedrights!)
exportCYGWIN="winsymlinks:native"
NotethatyoumustrunCygwinasadministratororchangetheWindowsgrouppoliciestoallowuseraccountstocreatesymlinks(gpedit.mscifavailable).
BTW:YoucancreatesymlinksmanuallyonWindowslike:
mklink/Htarget/file.extsource/file.ext
mklink/Dtarget/pathsource/path
mklink/Jtarget/pathsource/path/for/junction
AndinCygwin:
ln-ssourcetarget
MakingtheICUdatabasepublicallyavailableIfyouintendtousethemachinefordevelopmentpurposes,itmaybemorepracticaltocopyittoacommonplace:
cp3rdParty/V8/V8-5.0.71.39/third_party/icu/source/data/in/icudtl.dat/cygdrive/c/Windows/icudt54l.dat
Andconfigureyourenvironment(yesthisinstructionrememberstothehitchhikersguidetothegalaxy...)sothatICU_DATApointstoc:\\Windows.Youdothatbyopeningtheexplorer,rightclickonThisPCinthetreeontheleft,choosePropertiesintheopeningwindowAdvancedsystemsettings,inthePopupEnvironmentVariables,anotherpopupopens,intheSystemVariablespartyouclickNew,Andvariablename:ICU_DATAtothevalue:c:\\Windows
CompileonWindows
110
RunningUnitests(Optional)
Youcanthenruntheunittestsinthecygwinshelllikethat:
build64/bin/RelWithDebInfo/arangosh.exe\
-cetc/relative/arangosh.conf\
--log.levelwarning\
--server.endpointtcp://127.0.0.1:1024\
--javascript.executeUnitTests/unittest.js\
--\
all\
--rubyc:/tools/ruby22/bin/ruby\
--rspecc:/tools/ruby22/bin/rspec\
--buildTypeRelWithDebInfo\
--skipNondeterministictrue\
--skipTimeCriticaltrue
--skipBoosttrue\
--skipGeotrue
Documentation(Optional)NodeJS(needstoberunwithAdministratorprivilegesagain):
chocoinstall-ynodejs
Gitbook:
npminstall-ggitbook-cli
CompileonWindows
111
Markdown-pp:
gitclonehttps://github.com/triAGENS/markdown-pp.git
cdmarkdown-pp
pythonsetup.pyinstall
Ditaa:
Downloadandinstall:http://ditaa.sourceforge.net/#download
Authors:FrankCeller,WilfriedGoesgensandSimranBrucherseifer.
Tags:#windows
CompileonWindows
112
OpenSSLOpenSSL1.1isonitswaytomainstream.Sofar(ArangoDB3.2)hasonlybeenthorouglytestedwithOpenSSL1.0and1.1isunsupported.
Buildingagainst1.1willcurrentlyresultinacompileerror:
/arangodb/arangodb/lib/SimpleHttpClient/SslClientConnection.cpp:224:14:error:useofundeclaredidentifier'SSLv2_method'
meth=SSLv2_method();
^
/arangodb/arangodb/lib/SimpleHttpClient/SslClientConnection.cpp:239:14:warning:'TLSv1_method'isdeprecated[-Wdeprecated-dec
larations]
meth=TLSv1_method();
^
/usr/include/openssl/ssl.h:1612:45:note:'TLSv1_method'hasbeenexplicitlymarkeddeprecatedhere
DEPRECATEDIN_1_1_0(__owurconstSSL_METHOD*TLSv1_method(void))/*TLSv1.0*/
^
/arangodb/arangodb/lib/SimpleHttpClient/SslClientConnection.cpp:243:14:warning:'TLSv1_2_method'isdeprecated[-Wdeprecated-d
eclarations]
meth=TLSv1_2_method();
Youshouldinstallopenssl1.0(shouldbepossibletoinstallitalongside1.1).
Afterthathelpcmaketofindthe1.0variant.
ExampleonArchLinux:
cmake-DOPENSSL_INCLUDE_DIR=/usr/include/openssl-1.0/-DOPENSSL_SSL_LIBRARY=/usr/lib/libssl.so.1.0.0-DOPENSSL_CRYPTO_LIBRARY=/
usr/lib/libcrypto.so.1.0.0<SOURCE_PATH>
AfterthatArangoDBshouldcompilefine.
OpenSSL
113
Runningacustombuild
Problem
You'vealreadybuiltacustomversionofArangoDBandwanttorunit.Possiblyinisolationfromanexistinginstallationoryoumaywanttore-usethedata.
Solution
First,youneedtobuildyourownversionofArangoDB.Ifyouhaven'tdonesoalready,havealookatanyoftheCompilingrecipes.
Thisrecipeassumesyou'reintherootdirectoryoftheArangoDBdistributionandcompilinghassuccessfullyfinished.
Runninginisolation
Thispartshowshowtorunyourcustombuildwithanemptydatabasedirectory
#createdatadirectory
mkdir/tmp/arangodb
#run
bin/arangod\
--configurationetc/relative/arangod.conf\
--database.directory/tmp/arangodb
Runningwithdata
Thispartshowshowtorunyourcustombuildwiththeconfiganddatafromapre-existingstableinstallation.
BEWAREArangoDB'sdevelopersmaychangethedbfileformatandafterrunningwithachangedfileformat,theremaybenowayback.Alternativelyyoucanrunyourbuildinisolationanddumpandrestorethedatafromthestabletoyourcustombuild.
Whenrunninglikethis,youmustrunthedbasthearangoduser(thedefaultinstalledbythepackage)inordertohavewriteaccesstothelog,databasedirectoryetc.Runningasrootwilllikelymessupthefilepermissions-goodluckfixingthat!
#becomerootfirst
su
#nowswitchtoarangodandrun
su-arangod
bin/arangod--configuration/etc/arangodb/arangod.conf
Author:PatrickHuber
Tags:#build
RunningCustomBuild
114
JemallocThisarticleisonlyrelevantifyouintendtocompilearangodbonUbuntu16.10ordebiantesting
Onmoremodernlinuxsystems(development/floatingatthetimeofthiswriting)youmaygetcompile/linkerrorswitharangodbregardingjemalloc.ThisisduetocompilersswitchingtheirdefaultbehaviourregardingthePIC-PositionIndependendCode.Itseemscommonthatjemallocremainsinastagewherethischangeisn'tfollowedandcausesarangodbtoerroroutduringthelinkingphase.
Fromnowoncmakewilldetectthisandgiveyouthishint:
thestaticsystemjemallocisn'tsuitable!Recompilewiththecurrentcompilerordisableusing`-DCMAKE_CXX_FLAGS=-no-pie-DCM
AKE_C_FLAGS=-no-pie`
Nowyou'vegotthreechoices.
Doingwithoutjemalloc
Fixesthecompilationissue,butyouwillgetproblemswiththeglibcsheapfragmentationbehaviourwhichinthelongerrunwillleadtoaneverincreasingmemoryconsumptionofArangoDB.
So,whilethismaybesuitablefordevelopment/testingsystems,itsdefinitelynotforproduction.
DisablingPICaltogetherThiswillbuildanarangodwhichdoesn'tusethiscompilerfeature.Itmaybenotsonicefordevelopmentbuilds.Itcanbeachievedbyspecifyingtheseoptionsoncmake:
-DCMAKE_CXX_FLAGS=-no-pie-DCMAKE_C_FLAGS=-no-pie
Recompilejemalloc
Thesmartestwayistofixthejemalloclibrariespackagesonyoursystemsoitsreflectingthatnewbehaviour.Ondebian/ubuntusystemsitcanbeachievedlikethis:
apt-getinstallautomakedebhelperdocbook-xslxsltprocdpkg-dev
aptsourcejemalloc
cdjemalloc*
dpkg-buildpackage
cd..
dpkg-i*jemalloc*deb
Recompilingjemalloc
115
Cloud,DCOSandDocker
AmazonWebServices(AWS)
RunningonAWS
UpdateonAWS
MicrosoftAzureRunningonAzure
DockerDockerArangoDB
DockerwithNodeJSApp
GiantSwarm
IntheGiantSwarm
Mesos/DCOS
ArangoDBinMesos
DC/OS:Fullexample
Cloud,DCOSandDocker
116
RunningArangoDBonAWSArangoDBisavailableasAMIontheAWSMarketplace.
(Ifyou'vealreadyarunningArangoDBimageonAWSandneedanupdate,pleasehavealookatUpdatingArangoDBonAWS).
Hereisaquickguidehowtostart:
GotheArangoDBmarketplace,selectthelatestversionandclickonContinueUsethe1-ClickLaunchtabandselectthesizeoftheinstance(EC2InstanceType)youwishtouse.NowyoucancontinuewithaclickonAcceptTerms&Launchwith1-Click.
Note:Ifyoudonothaveakeypairawarningwillappearafterclickingandyouwillbeaskedtogenerateakeypair.
YousuccessfullylaunchedanArangoDBinstanceonAWS.
TheArangoDBWeb-InterfacecanbereachedusingtheAccessSoftwarebuttonorviapublicinstanceIPandthePort8529(e.g.:http://12.13.14.15:8529)ThedefaultuserisrootandthepasswordistheInstanceID(YoucanfindtheInstanceIDontheinstancelist).
RunningonAWS
117
IfyouwanttolearnmoreaboutArangoDB,startwiththe[ArangoDBFirstSteps][../../Manual/GettingStarted/index.html]inourDocumentationortryoneofourTutorialsorCookbookrecipes.
Author:IngoFriepoertner
Tags:#aws,#amazon,#howto
RunningonAWS
118
UpdatinganArangoDBImageonAWSIfyourunanArangoDBonAWSandusedtheprovidedAMIintheAWSMarketplace,youatsomepointwanttoupdatetothelatestrelease.TheprocesstosubmitandpublishanewArangoDBimagetothemarketplacetakessometimeandyoumightnotfindthelatestreleaseinthemarketplacestoreyet.
However,updatingtothelatestversionisnotthathard.
First,logintothevirtualmachinewiththeuserubuntuandthepublicDNSnameoftheinstance.
TostartanupdatetoaknownversionofArangoDByoucanuse:
sudoapt-getupdate
sudoapt-getinstallarangodb=2.5.7
ToupgradeanArangoDBinstancetoanewmajorversion(from2.5.xto2.6.x),use:
sudoapt-getinstallarangodb
Youmightgetawarningthattheconfigurationfilehaschanged:
Configurationfile'/etc/arangodb/arangod.conf'
==>Modified(byyouorbyascript)sinceinstallation.
==>Packagedistributorhasshippedanupdatedversion.
Whatwouldyouliketodoaboutit?Youroptionsare:
YorI:installthepackagemaintainer'sversion
NorO:keepyourcurrently-installedversion
D:showthedifferencesbetweentheversions
Z:startashelltoexaminethesituation
Thedefaultactionistokeepyourcurrentversion.
***arangod.conf(Y/I/N/O/D/Z)[default=N]?
Youshouldstaywiththecurrentconfiguration(type"N"),astherearesomechangesmadeintheconfigurationforAWS.Ifyoutype"Y"youwillloseaccessfromyourapplicationstothedatabasesomakesurethatdatabasedirectoryandserverendpointarevalid.
--server.database-directory
needstobe`/vol/...`forAWS
--server.endpoint
needstobe`tcp://0.0.0.0:8529`forAWS
Ifyouupdatetoanewmajorversion,youwillbeaskedtoupgradesothatadatabasemigrationcanbestarted:
sudoservicearangodbupgrade
sudoservicearangodbstart
NowArangoDBshouldbebacktonormal.
Fornowwehavetostickwiththismanualprocessbutemightcreateasimplerupdateprocessinthefuture.PleaseprovidefeedbackhowyouuseourAmazonAMIandhowwecanimproveyouruserexperience.
Author:IngoFriepoertner
Tags:#aws#upgrade
UpdateonAWS
119
ArangoDBinMicrosoftAzureIwanttouseArangoDBinMicrosoftAzure
Howto
Theshortansweris:goto
https://vmdepot.msopentech.com/
typein"ArangoDB",selecttheversionyourequireandpress"CreateVirtualMachine".
FollowtheinstructionsgiventhereandwithinminutesyouhavearunningArangoDBinstanceinMicrosoftAzure.Youwillreceiveanemailassoonasyourmachineisready.
Assumeyourmachineiscalledmyarangodb,thanyoucanaccessArangoDBpointingyourbrowserto
http://myarangodb.cloudapp.net:8529
Pleasenotethatforsecurityreasonsthedefaultinstanceispasswordprotected.
However,thepasswordfor"root"isempty.So,pleaseloginandchangethepasswordassoonaspossible.
Authors:FrankCeller
Tags:#azure,#howto
RunningonAzure
120
HowtorunArangoDBinaDockercontainer
Problem
HowdoyoumakeArangoDBruninaDockercontainer?
Solution
ArangoDBisnowavailableasanofficialrepositoryintheDockerHub(@seedocumentationthere).
Author:FrankCeller
Tags:#docker#howto
DockerArangoDB
121
ArangoDB,NodeJSandDocker
Problem
I'mlookingforaheadstartinusingtheArangoDBdockerimage.
Solution
WewillusetheguessergameforArangoDBfrom
https://github.com/arangodb/guesser
Thisisasimplegameguessinganimalsorthings.ItlearnswhileplayingandstoresthelearnedinformationinanArangoDBinstance.Thegameiswrittenusingtheexpressframework.
Note:Youneedtoswitchtothedockerbranch.
Thegamehasthetwocomponents
front-endwithnode.jsandexpressback-endwithArangoDBandFoxx
Thereforetheguessergameneedstwodockercontainers,onecontainerforthenode.jsservertorunthefront-endcodeandonecontainerforArangoDBforthestorageback-end.
NodeServer
ThegameisitselfcanbeinstallviaNPMorfromgithub.Thereisanimageavailablefromdockerhubcalledarangodb/example-guesserwhichisbasedontheDockerfilefromgithub.
Youcaneitherbuildthedockercontainerlocallyorsimplyusetheavailableonefromdockerhub.
unix>dockerrun-p8000:8000-enolink=1arangodb/example-guesser
Startingwithoutadatabaselink
UsingDB-Serverhttp://localhost:8529
Guesserappserverlisteningathttp://0.0.0.0:8000
Thiswillstart-upnodeandtheguessergameisavailableonport8000.Nowpointyourbrowsertoport8000.Youshouldseethestart-upscreen.However,withoutastoragebackenditwillbeprettyuseless.Therefore,stopthecontainerandproceedwiththenextstep.
Ifyouwanttobuildthecontainerlocally,checkouttheguessergamefrom
https://github.com/arangodb/example-guesser
Switchintothedocker/nodesubdirectoryandexecutedockerbuild..
ArangoDB
ArangoDBisalreadyavailableondocker,sowestartaninstance
unix>dockerrun--namearangodb-guesserarangodb/arangodb
showalloptions:
dockerrun-ehelp=1arangodb
startingArangoDBinstand-alonemode
DockerwithNodeJSApp
122
That'sit.Notethatinanproductiveenvironmentyouwouldneedtoattachastoragecontainertoit.Weignorethishereforthesakeofsimplicity.
GuesserGame
SomeTesting
UsetheguessergameimagetostarttheArangoDBshellandlinktheArangoDBinstancetoit.
unix>dockerrun--linkarangodb-guesser:db-link-itarangodb/example-guesserarangosh--server.endpoint@DB_LINK_PORT_8529_TCP
@
Theparameter--linkarangodb-guesser:db-linklinkstherunningArangoDBintotheapplicationcontainerandsetsanenvironmentvariableDB_LINK_PORT_8529_TCPwhichpointstotheexposedportoftheArangoDBcontainer:
DB_LINK_PORT_8529_TCP=tcp://172.17.0.17:8529
YourIPmayvary.Thecommandarangosh...attheendofdockercommandexecutestheArangoDBshellinsteadofthedefaultnodecommand.
Welcometoarangosh2.3.1[linux].Copyright(c)ArangoDBGmbH
UsingGoogleV83.16.14JavaScriptengine,READLINE6.3,ICU52.1
Prettyprintingvalues.
ConnectedtoArangoDB'tcp://172.17.0.17:8529'version:2.3.1,database:'_system',username:'root'
Type'tutorial'foratutorialor'help'toseecommonexamples
arangosh[_system]>
Theimportantlineis
ConnectedtoArangoDB'tcp://172.17.0.17:8529'version:2.3.1,database:'_system',username:'root'
Ittellsyouthattheapplicationcontainerwasabletoconnecttothedatabaseback-end.PressControl-Dtoexit.
StartUpTheGame
Readytoplay?Startthefront-endcontainerwiththedatabaselinkandinitializethedatabase.
unix>dockerrun--linkarangodb-guesser:db-link-p8000:8000-einit=1arangodb/example-guesser
Useyourbrowsertoplaythegameattheaddresshttp://127.0.0.1:8000/.The
-einit=1
isonlyneedthefirsttimeyoustart-upthefront-endandonlyonce.Thenexttimeyourunthefront-endorifyoustartasecondfront-endserveruse
unix>dockerrun--linkarangodb-guesser:db-link-p8000:8000arangodb/example-guesser
Author:FrankCeller
Tags:#docker
DockerwithNodeJSApp
123
ArangoDBintheGiantSwarmusingDockercontainers
Problem
IwanttouseArangoDBintheGiantSwarmwithDockercontainers.
Solution
GiantSwarmallowsyoutodescribeanddeployyourapplicationbyprovidingasimpleJSONdescription.Thecurrentweatherappisagoodexampleonhowtoinstallanapplicationwhichusestwocomponents,namelynodeandredis.
MycolleagueMaxhaswrittenaguessergamewithvariousfront-endsandArangoDBasbackend.InordertogetthefeelingofbeingpartoftheGiantSwarm,Ihavestartedtosetupthisgameintheswarm.
FirstSteps
Theguessergameconsistsofafront-endwrittenasexpressapplicationinnodeandastorageback-endusingArangoDBandasmallAPIdevelopedwithFoxx.
Thefront-endapplicationisavailableasimage
arangodb/example-guesser
andtheArangoDBback-endwiththeFoxxAPIas
arangodb/example-guesser-db
Thedockerfilesusedtocreatetheimagesareavailablefromgithub
https://github.com/arangodb/guesser
SetuptheSwarm
Setupyourswarmenvironmentasdescribedinthedocumentation.Createaconfigurationfilefortheswarmcalledarangodb.jsonandfireuptheapplication
{
"app_name":"guesser",
"services":[
{
"service_name":"guesser-game",
"components":[
{
"component_name":"guesser-front-end",
"image":"arangodb/example-guesser",
"ports":[8000],
"dependencies":[
{"name":"guesser-back-end","port":8529}
],
"domains":{"guesser.gigantic.io":8000}
},
{
"component_name":"guesser-back-end",
"image":"arangodb/example-guesser-db",
"ports":[8529]
}
]
}
]
}
IntheGiantSwarm
124
Thisdefinesanapplicationguesserwithasingleserviceguesser-game.Thisservicehastwocomponentsguesser-front-endandguesser-back-end.Thedockerimagesaredownloadedfromthestandarddockerrepository.
Theline
"domains":{"guesser.gigantic.io":8000}
exposestheinternalport8000totheexternalportonport80forthehostguesser.gigantic.io.
InordertotellGiantSwarmaboutyourapplication,execute
unix>swarmcreatearangodb.json
Creating'arangodb'inthe'fceller/dev'environment...
Appcreatedsuccessfully!
Thiswillcreateanapplicationcalledguesser.
unix>swarmstatusguesser
Appguesserisdown
servicecomponentinstanceidstatus
guesser-gameguesser-back-end5347e718-3d27-4356-b530-b24fc5d1e3f5down
guesser-gameguesser-front-end7cf25b43-13c4-4dd3-9a2b-a1e32c43ae0ddown
Weseethetwocomponentsofourapplication.Botharecurrentlypowereddown.
StartuptheGuesserGame
Startingyourenginesisnowonesimplecommand
unix>swarmstartguesser
Startingapplicationguesser...
Applicationguesserisup
Nowtheapplicationisup
unix>swarmstatusguesser
Appguesserisup
servicecomponentinstanceidstatus
guesser-gameguesser-back-end5347e718-3d27-4356-b530-b24fc5d1e3f5up
guesser-gameguesser-front-end7cf25b43-13c4-4dd3-9a2b-a1e32c43ae0dup
Pointyourbrowserto
http://guesser.gigantic.io
andguessananimal.
Ifyouwanttocheckthelogfilesofaninstanceyoucanasktheswarmgivingittheinstanceid.Forexample,theback-end
unix>swarmlogs5347e718-3d27-4356-b530-b24fc5d1e3f5
2014-12-1712:34:57.984554+0000UTC-systemd-StoppingUserguesser-back-end...
2014-12-1712:36:28.074673+0000UTC-systemd-5cfe11d6-343e-49bb-8029-06333844401f.servicestop-sigtermtimedout.Killing.
2014-12-1712:36:28.077821+0000UTC-systemd-5cfe11d6-343e-49bb-8029-06333844401f.service:mainprocessexited,code=killed
,status=9/KILL
2014-12-1712:36:38.213245+0000UTC-systemd-StoppedUserguesser-back-end.
2014-12-1712:36:38.213543+0000UTC-systemd-Unit5cfe11d6-343e-49bb-8029-06333844401f.serviceenteredfailedstate.
2014-12-1712:37:55.074158+0000UTC-systemd-StartingUserguesser-back-end...
2014-12-1712:37:55.208354+0000UTC-docker-Pullingrepositoryarangodb/example-guesser-db
2014-12-1712:37:56.995122+0000UTC-docker-Status:Imageisuptodateforarangodb/example-guesser-db:latest
2014-12-1712:37:57.000922+0000UTC-systemd-StartedUserguesser-back-end.
2014-12-1712:37:57.707575+0000UTC-docker--->startingArangoDB
IntheGiantSwarm
125
2014-12-1712:37:57.708182+0000UTC-docker--->waitingforArangoDBtobecomeready
2014-12-1712:38:28.157338+0000UTC-docker--->installingguessergame
2014-12-1712:38:28.59025+0000UTC-docker--->readyforbusiness
andthefront-end
unix>swarmlogs7cf25b43-13c4-4dd3-9a2b-a1e32c43ae0d
2014-12-1712:35:10.139684+0000UTC-systemd-StoppingUserguesser-front-end...
2014-12-1712:36:40.32462+0000UTC-systemd-aa7756a4-7a87-4633-bea3-e416d035188b.servicestop-sigtermtimedout.Killing.
2014-12-1712:36:40.327754+0000UTC-systemd-aa7756a4-7a87-4633-bea3-e416d035188b.service:mainprocessexited,code=killed
,status=9/KILL
2014-12-1712:36:50.567911+0000UTC-systemd-StoppedUserguesser-front-end.
2014-12-1712:36:50.568204+0000UTC-systemd-Unitaa7756a4-7a87-4633-bea3-e416d035188b.serviceenteredfailedstate.
2014-12-1712:38:04.796129+0000UTC-systemd-StartingUserguesser-front-end...
2014-12-1712:38:04.921273+0000UTC-docker-Pullingrepositoryarangodb/example-guesser
2014-12-1712:38:06.459366+0000UTC-docker-Status:Imageisuptodateforarangodb/example-guesser:latest
2014-12-1712:38:06.469988+0000UTC-systemd-StartedUserguesser-front-end.
2014-12-1712:38:07.391149+0000UTC-docker-UsingDB-Serverhttp://172.17.0.183:8529
2014-12-1712:38:07.613982+0000UTC-docker-Guesserappserverlisteningathttp://0.0.0.0:8000
ScalingUp
Yourgamebecomesasuccess.Well,scalingupthefront-endistrivial.
Simplychangeyourconfigurationfileandrecreatetheapplication:
{
"app_name":"guesser",
"services":[
{
"service_name":"guesser-game",
"components":[
{
"component_name":"guesser-front-end",
"image":"arangodb/example-guesser",
"ports":[8000],
"dependencies":[
{"name":"guesser-back-end","port":8529}
],
"domains":{"guesser.gigantic.io":8000},
"scaling_policy":{"min":2,"max":2}
},
{
"component_name":"guesser-back-end",
"image":"arangodb/example-guesser-db",
"ports":[8529]
}
]
}
]
}
Theimportantlineis
"scaling_policy":{"min":2,"max":2}
Ittellstheswarmtousetwofront-endcontainers.Inlaterversionoftheswarmyouwillbeabletochangethenumberofcontainersinarunningapplicationwiththecommand:
>swarmscaleupguesser/guesser-game/guesser-front-end--count=1
Scalingupcomponentguesser/guesser-game/guesser-front-endby1...
WeatArangoDBarehardatworktomakescalinguptheback-enddatabaseequallyeasy.Staytunedfornewreleasesinearly2015...
Authors:FrankCeller
Tags:#docker,#giantswarm,#howto
IntheGiantSwarm
126
ArangoDBonApacheMesosusingMarathonandDocker
Problem
IwanttouseArangoDBinApacheMesoswithDockercontainers.
Solution
MesosinitsnewestversionmakesitveryeasytouseArangoDB,becauseMesoshasaddedsupportfordockercontainers.TogetherwithMarathontostartthefront-endandback-endpartsofanapplication,installationisstraightforward.
MycolleagueMaxhaswrittenaguessergamewithvariousfront-endsandArangoDBasbackend.InordertogetthefeelingofbeingpartoftheMesosphere,IhavestartedtosetupthisgameinanDigitalOceanenvironment.
FirstSteps
Theguessergameconsistsofafront-endwrittenasexpressapplicationinnodeandastorageback-endusingArangoDBandasmallAPIdevelopedwiththeFoxxmicroservicesframework.
Thefront-endapplicationisavailableasimage
arangodb/example-guesser
andtheArangoDBback-endwiththeFoxxAPIas
arangodb/example-guesser-db
Thedockerfilesusedtocreatetheimagesareavailablefromgithub
https://github.com/arangodb/guesser
SetUptheEnvironment
FollowtheinstructionsonMesospheretosetupanenvironmentwithdockersupport.YoushouldendupwithsshaccesstotheMesosmaster.
SetUptheApplication
ForthistutorialwebindthedatabasetoafixedportontheMesosenvironment.Pleasenote,thatthemesosphereusesHAproxytomaptheglobalporttotherealhostandport.TheserverscreatedbyMesospherewillhaveaHAproxydefinedonallmastersandslaves.
Thatmeans,ifwechose32333asserviceportforthedatabase,itwillbereachableonthisportonallmastersandslaves.Theappdefinitionforthedatabaselookslike
{
"id":"/guesser/database",
"apps":[
{
"id":"/guesser/database/arangodb",
"container":{
"docker":{
"image":"arangodb/example-guesser-db",
"network":"BRIDGE",
"portMappings":[
{"containerPort":8529,"hostPort":0,"servicePort":32222,"protocol":"tcp"}
]
}
},
ArangoDBinMesos
128
"cpus":0.2,
"mem":512.0,
"instances":1
}
]
}
Thiswillstartthedockerimagefortheback-endandbindstheportto32222.
InsidethedockercontaineranenvironmentvariableHOSTissetbethemesosslavetopointtotheslave.Thefront-endcanthereforeaccessport32222onthishosttocontacttheHAproxy,gainingaccesstothedatabase.
Theappdefinitionforthefront-endlookslike
{
"id":"/guesser/frontend",
"apps":[
{
"id":"/guesser/frontend/node",
"container":{
"docker":{
"image":"arangodb/example-guesser",
"network":"BRIDGE",
"portMappings":[
{"containerPort":8000,"hostPort":0,"servicePort":32221,"protocol":"tcp"}
]
}
},
"cpus":0.2,
"mem":256.0,
"instances":1
}
]
}
Marathonallowstodefineagroupofapplicationswithdependenciesbetweenthecomponents.Thefront-enddependsontheback-end,thereforethecompletegroupdefinitionslookslike
{
"id":"/guesser",
"groups":[
{
"id":"/guesser/database",
"apps":[
{
"id":"/guesser/database/arangodb",
"container":{
"docker":{
"image":"arangodb/example-guesser-db",
"network":"BRIDGE",
"portMappings":[
{"containerPort":8529,"hostPort":0,"servicePort":32222,"protocol":"tcp"}
]
}
},
"cpus":0.2,
"mem":512.0,
"instances":1
}
]
},
{
"id":"/guesser/frontend",
"dependencies":["/guesser/database"],
"apps":[
{
"id":"/guesser/frontend/node",
"container":{
"docker":{
"image":"arangodb/example-guesser",
"network":"BRIDGE",
"portMappings":[
ArangoDBinMesos
129
{"containerPort":8000,"hostPort":0,"servicePort":32221,"protocol":"tcp"}
]
}
},
"cpus":0.2,
"mem":256.0,
"instances":1
}
]
}
]
}
Thisstartsoneinstanceoftheback-endcalled/guesser/database/arangodbandoneinstanceofthefront-endcalled/guesser/frontend/node.Thefront-enddependsontheback-end.
Inordertofireuptheguessergamesavetheabovedefinitioninafileguesser.jsonandexecute
curl-XPUT-H"Accept:application/json"-H"Content-Type:application/json"127.0.0.1:8080/v2/groups-d"`catguesser.json`"
onthemesosmaster.
IfyounowswitchtotheMarathonconsoleonport8080,youshouldseeapps,namely/guesser/database/arangodband/guesser/frontend/node.
Ifyouaccessport32222,youshouldseetheArangoDBconsole.
Andfinally,onport32211,youcanplaytheguessergame.
ArangoDBinMesos
130
ScalingUp
Yourgamebecomesasuccess.Well,scalingupthefront-endistrivial.Simply,gotothemarathonpageandscaleup/guesser/frontend/node.
Authors:FrankCeller
Tags:#docker,#mesos,#mesosphere,#howto
ArangoDBinMesos
131
DeployingahighlyavailableapplicationusingArangoDBandFoxxonDC/OS
Problem
HowcanIdeployanapplicationusingArangoDBonDC/OSandmakeeverythinghighlyavailable?
Solution
Toachievethisgoalseveralindividualcomponentshavetobecombined.
InstallDC/OS
Gotohttps://dcos.io/install/andfollowtheinstructionstoinstallDC/OS
AlsomakesuretoinstallthedcosCLI.
InstallArangoDB
OnceyourclusterisDC/OSclusterisreadyinstallthepackagearangodb3fromtheuniversetab(defaultsettingsarefine)
DetailedinstructionsmaybefoundinthefirstchaptersofourDC/OStutorialhere:
https://dcos.io/docs/1.7/usage/tutorials/arangodb/
TounderstandhowArangoDBensuresthatitishighlyavailablemakesuretoreadtheclusterdocumentationhere:
ArangoDBArchitectureDocumentation
Deployaloadbalancerforthecoordinators
OnceArangoDBisinstalledandhealthyyoucanaccesstheclusterviaoneofthecoordinators.
TodosofromtheoutsideDC/OSprovidesaniceandsecuregatewaythroughtheiradmininterface.
Howeverthisisintendedtobeusedfromtheoutsideonly.ApplicationsusingArangoDBasitsdatastorewillwanttoconnecttothecoordinatorsfromtheinside.YoucouldcheckthetasklistwithinDC/OStofindouttheendpointswherethecoordinatorsarelistening.Howeverthesearenottobetrusted:Theycanfailatanytime,thenumberofcoordinatorsmightchangeduetoup-anddownscalingorsomeonemightevenkillafullDC/OSAgentandtasksmaysimplyfailandreappearonadifferentendpoint.
Inshort:Endpointsaretemporary.
Tomitigatethisproblemwehavebuiltaspecialloadbalancerforthesecoordinators.
Toinstallit:
$gitclonehttps://github.com/arangodb/arangodb-mesos-haproxy
$cdarangodb-mesos-haproxy
$dcosmarathonappaddmarathon.json
Afterwardsyoucanusethefollowingendpointtoaccessthecoordinatorsfromwithinthecluster:
tcp://arangodb-proxy.marathon.mesos:8529
Tomakeithighlyavailableyoucansimplylaunchafewmoreinstancesusingthemarathonwebinterface.DetailsonhowtodothisandhowtodeployanapplicationusingtheUIcanbefoundhere:https://dcos.io/docs/1.7/usage/tutorials/marathon/marathon101/
Ourtestapplication
DC/OS:Fullexample
132
NowthatwehavesetupArangoDBonDC/OSitistimetodeployourapplication.Inthisexamplewewilluseourguesserapplicationwhichyoucanfindhere:
https://github.com/arangodb/guesser
ThisapplicationhassomeapplicationcodeandaFoxxmicroservice.
DeployingtheFoxxservice
OpentheArangoDBinterface(viatheServicestabintheDC/OSinterface)andgotoServices.
Enter/guesserasmountdirectoryChoosegithubonthetabandenterthefollowingrepository:
ArangoDB/guesser
Choosemasterasversion.
PressInstall
Deploytheapplication
Finallyitistimetodeploytheapplicationcode.Wehavepackagedeverythingintoadockercontainer.Theonlythingthatismissingissomeconnectioninfoforthedatabase.Thiscanbeprovidedviaenvironmentvariablesthroughmarathon.
OpenthemarathonwebinterfaceinyourDC/OScluster(Servicesandthenmarathon).
ThenclickCreateapplication
OnthetoprightyoucanchangetotheJSONview.Pastethefollowingconfig:
{
"id":"/guesser",
"cmd":null,
"cpus":1,
"mem":128,
"disk":0,
"instances":3,
"container":{
"type":"DOCKER",
"volumes":[],
"docker":{
"image":"arangodb/example-guesser",
"network":"BRIDGE",
"portMappings":[
{
"containerPort":8000,
"hostPort":0,
"servicePort":10004,
"protocol":"tcp"
}
],
"privileged":false,
"parameters":[],
"forcePullImage":true
}
},
"labels":{
"HAPROXY_GROUP":"external"
},
"env":{
"ARANGODB_SERVER":"http://arangodb-proxy.marathon.mesos:8529",
"ARANGODB_ENDPOINT":"tcp://arangodb-proxy.marathon.mesos:8529"
}
}
AsyoucanseeweareprovidingtheARANGODB_ENDPOINTasanenvironmentvariable.Thedockercontainerwilltakethatanduseitwhenconnecting.Thisconfigurationinjectionviaenvironmentvariablesisconsideredadockerbestpracticeandthisishowyoushouldprobablycreateyourapplicationsaswell.
DC/OS:Fullexample
133
Nowwehaveourguesserappstartedwithinmesos.
Itishighlyavailablerightawayaswelaunched3instances.ToscaleitupordownsimplyusethescalebuttonsinthemarathonUI.
Makeitpublicallyavailable
Forthistoworkweneedanothertool,namelymarathon-lb
Installit:
dcospackageinstallmarathon-lb
Afterinstallationitwillscanallmarathonapplicationsforaspecialsetoflabelsandmaketheseapplicationsavailabletothepublic.
Tomaketheguesserapplicationavailabletothepublicyoufirsthavetodetermineahostnamethatpointstotheexternalloadbalancerinyourenvironment.WheninstallingusingthecloudformationtemplateonAWSthisisthehostnameofthesocalledpublicslave.Youcanfinditintheoutputtabofthecloudformationtemplate.
Inmycasethiswas:
mop-publicslaveloa-3phq11mb7oez-1979417947.eu-west-1.elb.amazonaws.com
Incasethereareuppercasedletterspleasetakeextracaretolowercasethemasthemarathon-lbwillfailotherwise.
EditthesettingsoftheguesserappinthemarathonUIandaddthishostnameasthelabelHAPROXY_0_VHOSTeitherusingtheUIorusingtheJSONmode:
[...]
"labels":{
"HAPROXY_GROUP":"external",
"HAPROXY_0_VHOST":"mop-publicslaveloa-3phq11mb7oez-1979417947.eu-west-1.elb.amazonaws.com"
},
[...]
Toscaleitupandthusmakingithighlyavailableincreasetheinstancescountwithinmarathon.
FormoredetailedinformationandmoreconfigurationoptionsincludingSSLetcbesuretocheckthedocumentation:
https://docs.mesosphere.com/1.7/usage/service-discovery/marathon-lb/usage/
Accessingtheguessergame
Aftermarathon-lbhasreloadeditsconfiguration(whichshouldhappenalmostimmediately)youshouldbeabletoaccesstheguessergamebypointingawebbrowsertoyourhostname.
Havefun!
Conclusion
Therearequiteafewcomponentsinvolvedinthisprocessbutonceyouarefinishedyouwillhaveahighlyresilientsystemwhichsurelywasworththeeffort.Noneofthecomponentsinvolvedisasinglepointoffailure.
Author:AndreasStreichardt
Tags:#docker#howto#dcos#cluster#ha
DC/OS:Fullexample
134
RunningArangoDBonDC/OSwithMesosContainersSinceDC/OS1.8anewwayofrunningcontainersinMesoscloudshasbecomeavailable.Itre-usesthedockeron-diskformatanddistributioninfrastructure,butpairsitwithmanagementfeaturesthatmakeitabetterfitforDC/OSenvironments.
WithArangoDB3.2.6weintroducethepossibilitytoinstanciateanArangoDBClustersusingtheMesoscontainerizer.YoucandeployclusterswithitbyuncheckingtheUSEDOCKERcheckmark:
OncetheArangoDBframeworktaskisupandrunningyoucanrevalidateitsrunningusingtheMesoscontainerenginebyclickingonthetask,andscrollallthewaydownintheDetailstab:
UsingtheDC/OScliwecannowalsolisttherunningtasks:
DC/OS:ChoosingContainerengine
135
#dcostask
NAMEHOSTUSERSTATEIDMESOSID
arangodb310.0.1.221rootRarangodb3.988230ce-b95f-11e7-b0b3-d27390e16c964339f842-fb3b-46a6-9cb1-46febc
a9ad31-S4
arangodb3-Agent110.0.3.125rootRf1bbb380-6650-47c6-a6dd-31256b9db2a74339f842-fb3b-46a6-9cb1-46febc
a9ad31-S1
arangodb3-Agent210.0.0.234rootR410e4df2-5dea-4fae-9724-82e382488acd4339f842-fb3b-46a6-9cb1-46febc
a9ad31-S0
arangodb3-Agent310.0.0.231rootRbbb73025-00da-4bdf-8a6d-e34129e3abaf4339f842-fb3b-46a6-9cb1-46febc
a9ad31-S5
arangodb3-Coordinator110.0.3.125rootR9eea93a7-2ada-45c2-8bb6-f3f6153b7fd84339f842-fb3b-46a6-9cb1-46febc
a9ad31-S1
arangodb3-Coordinator210.0.0.234rootRc49496c2-ea66-4b75-9b0d-4d35e637ca774339f842-fb3b-46a6-9cb1-46febc
a9ad31-S0
arangodb3-DBServer110.0.0.234rootR43bdda44-4edb-457a-bde7-44d5711f076d4339f842-fb3b-46a6-9cb1-46febc
a9ad31-S0
arangodb3-DBServer210.0.3.125rootRff3ad9fb-d69a-4d1a-9bd7-43e782835d834339f842-fb3b-46a6-9cb1-46febc
a9ad31-S1
AndfindtherunningArangoDBcluster.WecannowusetheDC/OSclitogainashellontheframeworkcontainerbypickingitsIDfromthe5thcolumn:
dcostaskexec-itarangodb3.988230ce-b95f-11e7-b0b3-d27390e16c96bash
Whichwillgiveusaninteractiveshellinthatcontainer.Sincethecontainerisstrippeddowntothebareminimum,wemaywanttoinstallabunchoftoolsforbetterinspectingthecurrentstate:
root@ip-10-0-1-221:/mnt/mesos/sandbox#exportPATH=$PATH:/usr/sbin:/sbin;\
apt-getupdate;\
apt-getinstallcurlnet-toolsprocpsnetcatjq
Wethencani.e.inspecttherunningtasks:
root@ip-10-0-1-221:/mnt/mesos/sandbox#ps-eaf
UIDPIDPPIDCSTIMETTYTIMECMD
root10008:36?00:00:00/opt/mesosphere/active/mesos/libexec/mesos/mesos-containerizerlaunch
root61008:36?00:00:00mesos-executor--launcher_dir=/opt/mesosphere/active/mesos/libexec/mesos--sandbox_directory=
/mnt/mesos/sandbo
root166008:36?00:00:01./arangodb-framework--webui_port=10452--framework_port=10453--webui=http://10.0.1.221:1045
2framework--fra
root3816008:37?00:00:00haproxy-f/tmp/arango-haproxy.conf-sf37
root401008:42?00:00:00/opt/mesosphere/active/mesos/libexec/mesos/mesos-containerizerlaunch
root4140008:42?00:00:00bash
root46041008:44?00:00:00ps-eaf
DC/OS:ChoosingContainerengine
136
MonitoringArangoDBusingcollectd
Problem
TheArangoDBwebinterfaceshowsanicesummaryofthecurrentstate.IwanttoseesimilarnumbersinmymonitoringsystemsoIcananalyzethesystemusagepostmortemorsendalarmsonfailure.
Solution
CollectdisanexcellenttooltogatherallkindsofmetricsfromasystemanddeliverittoacentralmonitoringlikeGraphiteand/orNagios.
Ingredients
Forthisrecipeyouneedtoinstallthefollowingtools:
collectd>=5.4.2TheaggregationDaemonkcollectdforinspectingthedata
Configuringcollectd
ForaggregatingthevalueswewillusethecURL-JSONplug-in.WewillstorethevaluesusingtheRound-Robin-Databasewriter(RRD)whichkcollectdcanlateronpresenttoyou.
Weassumeyourcollectdcomesfromyourdistributionandreadsitsconfigfrom/etc/collectd/collectd.conf.Sincethisfiletendstobecomeprettyunreadablequickly,weusetheincludemechanism:
<Include"/etc/collectd/collectd.conf.d">
Filter"*.conf"
</Include>
Thiswaywecanmakeeachmetricgrouponcompactsetconfigfiles.Itconsistsofthreecomponents:
loadingtheplug-inaddingmetricstotheTypesDBtheconfigurationfortheplug-initself
rrdtool
WewillusetheRound-Robin-Databaseasstoragebackendfornow.Itcreatesitsowndatabasefilesoffixedsizeforeachspecifictimerange.Lateryoumaychoosemoreadvancedwriter-plug-ins,whichmaydonetworkdistributionofyourmetricsorintegratetheabovementionedGraphiteoryouralreadyestablishedmonitoring,etc.
FortheRRDwewillgoprettymuchwithdefaults:
#Loadtheplug-in:
LoadPluginrrdtool
<Pluginrrdtool>
DataDir"/var/lib/collectd/rrd"
#CacheTimeout120
#CacheFlush900
#WritesPerSecond30
#CreateFilesAsyncfalse
#RandomTimeout0
#
#Thefollowingsettingsareratheradvanced
#andshouldusuallynotbetouched:
#StepSize10
#HeartBeat20
Monitoring
137
#RRARows1200
#RRATimespan158112000
#XFF0.1
</Plugin>
cURLJSON
Collectdcomeswithawiderangeofmetricaggregationplug-ins.ManytoolstodayuseJSONasdataformatinggrammar;sodoesArangoDB.Thereforeaplug-inofferingtofetchJSONdocumentsviaHTTPistheperfectmatchasanintegrationinterface:
#Loadtheplug-in:
LoadPlugincurl_json
#weneedtouseourowntypestogenerateindividualnamesforourgauges:
TypesDB"/etc/collectd/collectd.conf.d/arangodb_types.db"
<Plugincurl_json>
#AdjusttheURLsocollectdcanreachyourarangod:
<URL"http://localhost:8529/_db/_system/_admin/aardvark/statistics/short">
#SetyourauthenticationtoAardvarkhere:
#User"foo"
#Password"bar"
<Key"totalTimeDistributionPercent/values/0">
Type"totalTimeDistributionPercent_values"
</Key>
<Key"totalTimeDistributionPercent/cuts/0">
Type"totalTimeDistributionPercent_cuts"
</Key>
<Key"requestTimeDistributionPercent/values/0">
Type"requestTimeDistributionPercent_values"
</Key>
<Key"requestTimeDistributionPercent/cuts/0">
Type"requestTimeDistributionPercent_cuts"
</Key>
<Key"queueTimeDistributionPercent/values/0">
Type"queueTimeDistributionPercent_values"
</Key>
<Key"queueTimeDistributionPercent/cuts/0">
Type"queueTimeDistributionPercent_cuts"
</Key>
<Key"bytesSentDistributionPercent/values/0">
Type"bytesSentDistributionPercent_values"
</Key>
<Key"bytesSentDistributionPercent/cuts/0">
Type"bytesSentDistributionPercent_cuts"
</Key>
<Key"bytesReceivedDistributionPercent/values/0">
Type"bytesReceivedDistributionPercent_values"
</Key>
<Key"bytesReceivedDistributionPercent/cuts/0">
Type"bytesReceivedDistributionPercent_cuts"
</Key>
<Key"numberOfThreadsCurrent">
Type"gauge"
</Key>
<Key"numberOfThreadsPercentChange">
Type"gauge"
</Key>
<Key"virtualSizeCurrent">
Type"gauge"
</Key>
<Key"virtualSizePercentChange">
Type"gauge"
</Key>
<Key"residentSizeCurrent">
Type"gauge"
</Key>
<Key"residentSizePercent">
Type"gauge"
</Key>
<Key"asyncPerSecondCurrent">
Type"gauge"
</Key>
<Key"asyncPerSecondPercentChange">
Type"gauge"
Monitoring
138
</Key>
<Key"syncPerSecondCurrent">
Type"gauge"
</Key>
<Key"syncPerSecondPercentChange">
Type"gauge"
</Key>
<Key"clientConnectionsCurrent">
Type"gauge"
</Key>
<Key"clientConnectionsPercentChange">
Type"gauge"
</Key>
<Key"physicalMemory">
Type"gauge"
</Key>
<Key"nextStart">
Type"gauge"
</Key>
<Key"waitFor">
Type"gauge"
</Key>
<Key"numberOfThreads15M">
Type"gauge"
</Key>
<Key"numberOfThreads15MPercentChange">
Type"gauge"
</Key>
<Key"virtualSize15M">
Type"gauge"
</Key>
<Key"virtualSize15MPercentChange">
Type"gauge"
</Key>
<Key"asyncPerSecond15M">
Type"gauge"
</Key>
<Key"asyncPerSecond15MPercentChange">
Type"gauge"
</Key>
<Key"syncPerSecond15M">
Type"gauge"
</Key>
<Key"syncPerSecond15MPercentChange">
Type"gauge"
</Key>
<Key"clientConnections15M">
Type"gauge"
</Key>
<Key"clientConnections15MPercentChange">
Type"gauge"
</Key>
</URL>
</Plugin>
Tocircumventtheshortcomingofthecurl_JSONplug-intoonlytakethelastpathelementasnameforthemetric,weneedtogivethemanameusingourowntypes.dbfilein/etc/collectd/collectd.conf.d/arangodb_types.db:
totalTimeDistributionPercent_valuesvalue:GAUGE:U:U
totalTimeDistributionPercent_cutsvalue:GAUGE:U:U
requestTimeDistributionPercent_valuesvalue:GAUGE:U:U
requestTimeDistributionPercent_cutsvalue:GAUGE:U:U
queueTimeDistributionPercent_valuesvalue:GAUGE:U:U
queueTimeDistributionPercent_cutsvalue:GAUGE:U:U
bytesSentDistributionPercent_valuesvalue:GAUGE:U:U
bytesSentDistributionPercent_cutsvalue:GAUGE:U:U
bytesReceivedDistributionPercent_valuesvalue:GAUGE:U:U
bytesReceivedDistributionPercent_cutsvalue:GAUGE:U:U
Rollingyourown
YoumaywanttomonitoryourownmetricsfromArangoDB.Hereisasimpleexamplehowtousetheconfig:
Monitoring
139
{
"testArray":[1,2],
"testArrayInbetween":[{"blarg":3},{"blub":4}],
"testDirectHit":5,
"testSubLevelHit":{"oneMoreLevel":6}
}
ThisconfigsnippetwillparsetheJSONabove:
<Key"testArray/0">
Type"gauge"
#Expect:1
</Key>
<Key"testArray/1">
Type"gauge"
#Expect:2
</Key>
<Key"testArrayInbetween/0/blarg">
Type"gauge"
#Expect:3
</Key>
<Key"testArrayInbetween/1/blub">
Type"gauge"
#Expect:4
</Key>
<Key"testDirectHit">
Type"gauge"
#Expect:5
</Key>
<Key"testSubLevelHit/oneMoreLevel">
Type"gauge"
#Expect:6
</Key
Getitserved
Nowwewill(re)startcollectdsoitpicksupourconfiguration:
/etc/init.d/collectdstart
Wewillinspectthesyslogtorevalidatenothingwentwrong:
Mar313:59:52localhostcollectd[11276]:Startingstatisticscollectionandmonitoringdaemon:collectd.
Mar313:59:52localhostsystemd[1]:StartedLSB:managethestatisticscollectiondaemon.
Mar313:59:52localhostcollectd[11283]:Initializationcomplete,enteringread-loop.
Collectdaddsthehostnametothedirectoryaddress,sonowweshouldhavefileslikethese:
-rw-r--r--1rootroot154888Mar216:53/var/lib/collectd/rrd/localhost/curl_json-default/gauge-numberOfThreads15M.rrd
NowwestartkcollectdtoviewthevaluesintheRRDfile:
Monitoring
140
Sincewestartedputtingvaluesinjustnow,weneedtochoose'lasthour'andzoominalittlemoretoinspectthevalues.
Finishedwiththisdish,waitformoremetricstocomeinotherrecipes.
Author:WilfriedGoesgens
Tags:#json#monitoring
Monitoring
141
MonitoringreplicationslaveNote:thisrecipeisworkingwithArangoDB2.5,youneedacollectdcurl_jsonpluginwithcorrectbooleantypemapping.
Problem
Howtomonitortheslavestatususingthecollectdcurl_JSONplugin.
Solution
SincearangodbreportsthereplicationstatusinJSON,integratingitwiththecollectdcurl_JSONpluginshouldbeaneasyexercise.However,onlyveryrecentversionsofcollectdwillhandlebooleanflagscorrectly.
Ourtestmaster/slavesetuprunswiththethemasterlisteningontcp://127.0.0.1:8529andtheslave(whichwequery)listeningontcp://127.0.0.1:8530.TheyreplicateadabatasebythenametestDatabase.
Sincereplicationappliersareactiveperdatabaseandourexampledoesn'tusethedefault_system,weneedtospecifyitsnameintheURLlikethis:_db/testDatabase.
Weneedtoparseadocumentfromarequestlikethis:
curl--dump-http://localhost:8530/_db/testDatabase/_api/replication/applier-state
Ifthereplicationisnotrunningthedocumentwilllooklikethat:
{
"state":{
"running":false,
"lastAppliedContinuousTick":null,
"lastProcessedContinuousTick":null,
"lastAvailableContinuousTick":null,
"safeResumeTick":null,
"progress":{
"time":"2015-11-02T13:24:07Z",
"message":"appliershutdown",
"failedConnects":0
},
"totalRequests":1,
"totalFailedConnects":0,
"totalEvents":0,
"totalOperationsExcluded":0,
"lastError":{
"time":"2015-11-02T13:24:07Z",
"errorMessage":"nostarttick",
"errorNum":1413
},
"time":"2015-11-02T13:31:53Z"
},
"server":{
"version":"2.7.0",
"serverId":"175584498800385"
},
"endpoint":"tcp://127.0.0.1:8529",
"database":"testDatabase"
}
Arunningreplicationwillreturnsomethinglikethis:
{
"state":{
"running":true,
"lastAppliedContinuousTick":"1150610894145",
"lastProcessedContinuousTick":"1150610894145",
Collectd-ReplicationSlaves
142
"lastAvailableContinuousTick":"1151639153985",
"safeResumeTick":"1150610894145",
"progress":{
"time":"2015-11-02T13:49:56Z",
"message":"fetchingmasterlogfromtick1150610894145",
"failedConnects":0
},
"totalRequests":12,
"totalFailedConnects":0,
"totalEvents":2,
"totalOperationsExcluded":0,
"lastError":{
"errorNum":0
},
"time":"2015-11-02T13:49:57Z"
},
"server":{
"version":"2.7.0",
"serverId":"175584498800385"
},
"endpoint":"tcp://127.0.0.1:8529",
"database":"testDatabase"
}
Wecreateasimplecollectdconfigurationin/etc/collectd/collectd.conf.d/slave_testDatabase.confthatmatchesourAPI:
TypesDB"/etc/collectd/collectd.conf.d/slavestate_types.db"
<Plugincurl_json>
#AdjusttheURLsocollectdcanreachyourarangodslaveinstance:
<URL"http://localhost:8530/_db/testDatabase/_api/replication/applier-state">
#Setyourauthenticationtothatdatabasehere:
#User"foo"
#Password"bar"
<Key"state/running">
Type"boolean"
</Key>
<Key"state/totalOperationsExcluded">
Type"counter"
</Key>
<Key"state/totalRequests">
Type"counter"
</Key>
<Key"state/totalFailedConnects">
Type"counter"
</Key>
</URL>
</Plugin>
Togetnicemetricnames,wespecifyourowntypes.dbfilein/etc/collectd/collectd.conf.d/slavestate_types.db:
booleanvalue:ABSOLUTE:0:1
So,basicallystate/runningwillgiveyou0/1ifits(not/)runningthroughthecollectdmonitor.
Author:WilfriedGoesgens
Tags:#monitoring#foxx#json
Collectd-ReplicationSlaves
143
MonitoringArangoDBClusternetworkusage
Problem
Werunaclusterandwanttoknowwhetherthetrafficisunbalancedorsomethinglikethat.Wewantacheapestimatewhichhosthashowmuchtraffic.
Solution
AswealreadyrunCollectdasourmetric-hub,wewanttoutilizeittoalsogiveusthesefigures.AverycheapwaytogeneratethesevaluesarethecountersintheIPTablesfirewallofoursystem.
Ingredients
Forthisrecipeyouneedtoinstallthefollowingtools:
collectd:theaggregationDaemonkcollectdforinspectingthedataiptables-shouldcomewithyourLinuxdistributionfermforcompactfirewallcodewebaseonMonitoringwithCollecdrecipeforunderstandingthebasicsaboutcollectd
GettingthestateandthePortsofyourcluster
Nowweneedtofindoutthecurrentconfigurationofourcluster.Forthetimebeingweassumeyousimplyissued
./scripts/startLocalCluster.sh
togetyousetup.Soyouknowyou'vegottwoDB-Servers-oneCoordinator,oneagent:
ps-eaf|greparango
arangod214061116:59pts/1400:00:00bin/etcd-arango--data-dir/var/tmp/tmp-21550-1347489353/shell_server/agentar
ango4001--nameagentarango4001--bind-addr127.0.0.1:4001--addr127.0.0.1:4001--peer-bind-addr127.0.0.1:7001--peer-addr12
7.0.0.1:7001--initial-cluster-statenew--initial-clusteragentarango4001=http://127.0.0.1:7001
arangod214081416:56pts/1400:00:01bin/arangod--database.directorycluster/data8629--cluster.agency-endpointt
cp://localhost:4001--cluster.my-addresstcp://localhost:8629--server.endpointtcp://localhost:8629--cluster.my-local-infodb
server:localhost:8629--log.filecluster/8629.log--cluster.my-idPavel
arangod214101516:56pts/1400:00:02bin/arangod--database.directorycluster/data8630--cluster.agency-endpointt
cp://localhost:4001--cluster.my-addresstcp://localhost:8630--server.endpointtcp://localhost:8630--cluster.my-local-infodb
server:localhost:8630--log.filecluster/8630.log--cluster.my-idPerry
arangod214161516:56pts/1400:00:02bin/arangod--database.directorycluster/data8530--cluster.agency-endpointt
cp://localhost:4001--cluster.my-addresstcp://localhost:8530--server.endpointtcp://localhost:8530--cluster.my-local-infoco
ordinator:localhost:8530--log.filecluster/8530.log--cluster.my-idClaus
Wecannowcheckwhichportstheyoccupied:
netstat-aplnt|greparango
tcp00127.0.0.1:70010.0.0.0:*LISTEN21406/etcd-arango
tcp00127.0.0.1:40010.0.0.0:*LISTEN21406/etcd-arango
tcp00127.0.0.1:85300.0.0.0:*LISTEN21416/arangod
tcp00127.0.0.1:86290.0.0.0:*LISTEN21408/arangod
tcp00127.0.0.1:86300.0.0.0:*LISTEN21410/arangod
Theagenthas7001and4001.Sinceit'srunninginsingleservermodeitsclusterport(7001)shouldnotshowanytraffic,port4001istheinterestingone.Claus-Thisisthecoordinator.YourApplicationwilltalktoitonport8530Pavel-ThisisthefirstDB-Server;Clauswilltalktoitonport8629Perry-ThisisthesecondDB-Server;Clauswilltalktoitonport8630
Collectd-Networkusage
144
ConfiguringIPTables/ferm
SincetheusualsolutionusingshellscriptscallingiptablesbringstheDRYprincipletoagrindinghold,weneedsomethingbetter.Herefermcomestotherescue-Itenablesyoutoproduceverycompactandwellreadablefirewallconfigurations.
Accordingtotheportswefoundinthelastsection,wewillconfigureourfirewallin/etc/ferm/ferm.conf,andputtheidentitiesintothecommentssowehaveapersistentnamingscheme:
#blindlyforwardthesetotheaccountingchain:
@def$ARANGO_RANGE=4000:9000;
@def&TCP_ACCOUNTING($PORT,$COMMENT,$SRCCHAIN)={
@def$FULLCOMMENT=@cat($COMMENT,"_",$SRCCHAIN);
dport$PORTmodcommentcomment$FULLCOMMENTNOP;
}
@def&ARANGO_ACCOUNTING($CHAINNAME)={
#Thecoordinators:
&TCP_ACCOUNTING(8530,"Claus",$CHAINNAME);
#Thedb-servers:
&TCP_ACCOUNTING(8629,"Pavel",$CHAINNAME);
&TCP_ACCOUNTING(8630,"Perry",$CHAINNAME);
#Theagency:
&TCP_ACCOUNTING(4001,"etcd_client",$CHAINNAME);
#itshouldn'ttalktoitselfifitisonlyrunningwithasingleinstance:
&TCP_ACCOUNTING(7007,"etcd_cluster",$CHAINNAME);
}
tablefilter{
chainINPUT{
prototcpdport$ARANGO_RANGE@subchain"Accounting"{
&ARANGO_ACCOUNTING("input");
}
policyDROP;
#connectiontracking
modstatestateINVALIDDROP;
modstatestate(ESTABLISHEDRELATED)ACCEPT;
#allowlocalpacket
interfaceloACCEPT;
#respondtoping
protoicmpACCEPT;
#allowIPsec
protoudpdport500ACCEPT;
proto(espah)ACCEPT;
#allowSSHconnections
prototcpdportsshACCEPT;
}
chainOUTPUT{
policyACCEPT;
prototcpdport$ARANGO_RANGE@subchain"Accounting"{
&ARANGO_ACCOUNTING("output");
}
#connectiontracking
#modstatestateINVALIDDROP;
modstatestate(ESTABLISHEDRELATED)ACCEPT;
}
chainFORWARD{
policyDROP;
#connectiontracking
modstatestateINVALIDDROP;
modstatestate(ESTABLISHEDRELATED)ACCEPT;
}
}
Note:Thisisaverybasicconfiguration,mainlywiththepurposetodemonstratetheaccountingfeature-sodon'trunthisinproduction)
Collectd-Networkusage
145
Afteractivatingitinteractivelywith
ferm-i/etc/ferm/ferm.conf
Wenowusetheiptablescommandlineutilitydirectlytoreviewthestatusourcurrentsetting:
iptables-L-nvx
ChainINPUT(policyDROP85packets,6046bytes)
pktsbytestargetprotoptinoutsourcedestination
76361821798Accountingtcp--**0.0.0.0/00.0.0.0/0tcpdpts:4000:9000
00DROPall--**0.0.0.0/00.0.0.0/0stateINVALID
1470014857709ACCEPTall--**0.0.0.0/00.0.0.0/0stateRELATED,ESTABLISHED
1307800ACCEPTall--lo*0.0.0.0/00.0.0.0/0
00ACCEPTicmp--**0.0.0.0/00.0.0.0/0
00ACCEPTudp--**0.0.0.0/00.0.0.0/0udpdpt:500
00ACCEPTesp--**0.0.0.0/00.0.0.0/0
00ACCEPTah--**0.0.0.0/00.0.0.0/0
00ACCEPTtcp--**0.0.0.0/00.0.0.0/0tcpdpt:22
ChainFORWARD(policyDROP0packets,0bytes)
pktsbytestargetprotoptinoutsourcedestination
00DROPall--**0.0.0.0/00.0.0.0/0stateINVALID
00ACCEPTall--**0.0.0.0/00.0.0.0/0stateRELATED,ESTABLISHED
ChainOUTPUT(policyACCEPT296packets,19404bytes)
pktsbytestargetprotoptinoutsourcedestination
77201882404Accountingtcp--**0.0.0.0/00.0.0.0/0tcpdpts:4000:9000
1457514884356ACCEPTall--**0.0.0.0/00.0.0.0/0stateRELATED,ESTABLISHED
ChainAccounting(2references)
pktsbytestargetprotoptinoutsourcedestination
20457750tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8530/*Claus_input*/
2017890tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8629/*Pavel_input*/
26297352tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8630/*Perry_input*/
2604336184tcp--**0.0.0.0/00.0.0.0/0tcpdpt:4001/*etcd_client_inpu
t*/
00tcp--**0.0.0.0/00.0.0.0/0tcpdpt:7007/*etcd_cluster_inp
ut*/
20457750tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8530/*Claus_output*/
2017890tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8629/*Pavel_output*/
26297352tcp--**0.0.0.0/00.0.0.0/0tcpdpt:8630/*Perry_output*/
2604336184tcp--**0.0.0.0/00.0.0.0/0tcpdpt:4001/*etcd_client_outp
ut*/
00tcp--**0.0.0.0/00.0.0.0/0tcpdpt:7007/*etcd_cluster_out
put*/
YoucanseenicelytheAccountingsub-chainwithourcomments.Theseshouldbeprettystraightforwardtomatch.Wealsoseethepktsandbytescolumns.Theycontainthecurrentvalueofthesecountersofyoursystem.
Readmoreaboutlinuxfirewallingandfermconfigurationtobesureyoudotherightthing.
ConfiguringCollectdtopickupthesevalues
Sinceyoursystemnowgeneratesthesenumbers,wewanttoconfigurecollectdwithitsiptablesplugintoaggregatethem.
Wedosointhe/etc/collectd/collectd.conf.d/iptables.conf:
LoadPluginiptables
<Pluginiptables>
Chainfilter"Accounting""Claus_input"
Chainfilter"Accounting""Pavel_input"
Chainfilter"Accounting""Perry_input"
Chainfilter"Accounting""etcd_client_input"
Chainfilter"Accounting""etcd_cluster_input"
Chainfilter"Accounting""Claus_output"
Chainfilter"Accounting""Pavel_output"
Chainfilter"Accounting""Perry_output"
Chainfilter"Accounting""etcd_client_output"
Chainfilter"Accounting""etcd_cluster_output"
</Plugin>
Collectd-Networkusage
146
Nowwerestartcollectdwith/etc/init.d/collectdrestart,watchthesyslogforerrors.IfeverythingisOK,ourvaluesshouldshowupin:
/var/lib/collectd/rrd/localhost/iptables-filter-Accounting/ipt_packets-Claus_output.rrd
Wecaninspectourvalueswithkcollectd:
Author:WilfriedGoesgens
Tags:#monitoring
Collectd-Networkusage
147
MonitoringotherrelevantmetricsofArangoDB
Problem
AsideofthevalueswhichArangoDBalreadyoffersformonitoring,othersystemmetricsmayberelevantforcontinuouslyoperatingArangoDB.beitasingleinstanceoraclustersetup.Collectdoffersapleathoraofplugins-letshavealookatsomeofthemwhichmaybeusefulforus.
Solution
Ingedients
Forthisrecipeyouneedtoinstallthefollowingtools:
collectd:ThemetricsaggregationDaemonwebaseonMonitoringwithCollecdrecipeforunderstandingthebasicsaboutcollectd
Diskusage
YoumaywanttomonitorthatArangoDBdoesn'trunoutofdiskspace.ThedfPlugincanaggregatethesevaluesforyou.
FirstweneedtofindoutwhichdisksareusedbyyourArangoDB.Bydefaultyouneedtofind/var/lib/arangointhemountpoints.Sincenowadaysmanyvirtualfilesystemsarealsomountedonatypical*nixsystemwewanttosorttheoutputofmount:
mount|sort
/dev/sda3on/local/hometypeext4(rw,relatime,data=ordered)
/dev/sda4on/typeext4(rw,relatime,data=ordered)
/dev/sdb1on/mnttypevfat(rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=utf8,shortname=mixed,errors=remount-ro)
binfmt_miscon/proc/sys/fs/binfmt_misctypebinfmt_misc(rw,relatime)
cgroupon/sys/fs/cgroup/blkiotypecgroup(rw,nosuid,nodev,noexec,relatime,blkio)
....
udevon/devtypedevtmpfs(rw,relatime,size=10240k,nr_inodes=1022123,mode=755)
Soherewecanseethemountpointsare/,/local/home,/mnt/so/var/lib/canbefoundontherootpartition(/)/dev/sda3here.AproductionsetupmaybedifferentsotheOSdoesn'tinterferewiththeservices.
Thecollectdconfiguration/etc/collectd/collectd.conf.d/diskusage.conflookslikethis:
LoadPlugindf
<Plugindf>
Device"/dev/sda3"
#Device"192.168.0.2:/mnt/nfs"
#MountPoint"/home"
#FSType"ext4"
#ignorerootfs;else,therootfile-systemwouldappeartwice,causing
#oneoftheupdatestofailandspamthelog
FSTyperootfs
#ignoretheusualvirtual/temporaryfile-systems
FSTypesysfs
FSTypeproc
FSTypedevtmpfs
FSTypedevpts
FSTypetmpfs
FSTypefusectl
FSTypecgroup
IgnoreSelectedtrue
#ReportByDevicefalse
#ReportReservedfalse
#ReportInodesfalse
#ValuesAbsolutetrue
#ValuesPercentagefalse
</Plugin>
Collectd-moreMetrics
148
DiskI/OUsage
Anotherinterestingmetricistheamountofdataread/writtentodisk-itsanestimatehowbusyyourArangoDBorthewholesystemcurrentlyis.TheDiskpluginaggregatesthesevalues.
Accordingtothemountpointsaboveourconfiguration/etc/collectd/collectd.conf.d/disk_io.conflookslikethis:
LoadPlugindisk
<Plugindisk>
Disk"hda"
Disk"/sda[23]/"
IgnoreSelectedfalse
</Plugin>
CPUUsage
WhiletheArangoDBselfmonitoringalreadyofferssomeoverviewoftherunningthreadsetc.youcangetadeeperviewusingtheProcessPlugin.
Ifyou'rerunningasingleArangoinstance,asimplematchbyprocessnameissufficient,/etc/collectd/collectd.conf.d/arango_process.conflookslikethis:
LoadPluginprocesses
<Pluginprocesses>
Process"arangod"
</Plugin>
Ifyou'rerunningacluster,youcanmatchthespecificinstancesbycommand-lineparameters,/etc/collectd/collectd.conf.d/arango_cluster.conflookslikethis:
LoadPluginprocesses
<Pluginprocesses>
ProcessMatch"Claus""/usr/bin/arangod.*--cluster.my-idClaus.*"
ProcessMatch"Pavel""/usr/bin/arangod.*--cluster.my-idPavel.*"
ProcessMatch"Perry""/usr/bin/arangod.*--cluster.my-idPerry.*"
Process"etcd-arango"
</Plugin>
MorePlugins
Asmentionedabove,thelistofavailablepluginsishuge;Herearesomemoreonecouldbeinterestedin:
usetheCPUPlugintomonitortheoverallCPUutilizationusetheMemoryPlugintomonitormainmemoryavailabilityusetheSwapPlugintoseewhetherexcessRAMusageforcesthesystemtopageandthusslowdownEthernetStatisticswithwhatsgoingonatyourNetworkcardstogetamorebroadoverviewofnetworktrafficyoumayTaillogfileslikeanapacherequestlogandpickspecificrequestsbyregularexpressionsParsetabularfilesinthe/procfilesystemyoucanusefilterstoreducetheamountofdatacreatedbyplugins(i.e.ifyouhavemanyCPUcores,youmaywantthecombinedresult).Itcanalsodecidewheretoroutedataandtowhichwriterpluginwhileyoumayhaveseenthatmetricsarestoredatafixedrateorfrequency,yourmetrics(i.e.thedurationsofwebrequests)maycomeinarandom&higherfrequency.Thusyouwanttoburnthemdowntoafixedfrequency,andknowMin/Max/Average/Median.SoyouwanttoAggregatevaluesusingthestatsdpattern.YoumaystartrollingyourowninPython,java,PerlorforsureinC,thelanguagecollectdisimplementedin
Finallywhilekcollectdisnicetogetaquicksuccessatinspectingyourcollectedmetricsduringworkingyourwayintocollectd,itsnotassufficientforoperatingaproductionsite.SincecollectdsdefaultstorageRRDisalreadywidespreadinsystemmonitoring,therearemanywebfrontentstochooseforthevisualization.SomeofthemreplacetheRRDstoragebysimplyaddingawriterplugin,mostprominenttheGraphitegraphingframeworkwiththeGraphitewriterwhichallowsyoutocombinerandommetricsinsinglegraphs-tofindcoincidencesinyourdatayouneverdreamedof.
IfyoualreadyrunNagiosyoucanusetheNagiostooltosubmitvalues.
Collectd-moreMetrics
149
Wehopeyounowhaveagoodoverviewofwhatspossible,butasusualitsagoodideatobrowsetheFineManual.
Author:WilfriedGoesgens
Tags:#monitoring
Collectd-moreMetrics
150
MonitoringyourFoxxapplicationsNote:thisrecipeisworkingwithArangoDB2.5Foxx
Problem
HowtointegrateaFoxxapplicationintoamonitoringsystemusingthecollectdcurl_JSONplugin.
Solution
SinceFoxxnativetongueisJSON,integratingitwiththecollectdcurl_JSONpluginshouldbeaneasyexercise.WehaveaFoxx-ApplicationwhichcanreceiveDataandwriteitintoacollection.WespecifyaneasyinputModel:
Model=Foxx.Model.extend({
schema:{
//DescribetheattributeswithJoihere
'_key':Joi.string(),
'value':Joi.number()
}
});
AnduseasimpleFoxx-Routetoinjectdataintoourcollection:
/**CreatesanewFirstCollection
*
*CreatesanewFirstCollection-Item.Theinformationhastobeinthe
*requestBody.
*/
controller.post('/firstCollection',function(req,res){
varfirstCollection=req.params('firstCollection');
firstCollection.attributes.Date=Date.now();
res.json(FirstCollection_repo.save(firstCollection).forClient());
}).bodyParam('firstCollection',{
description:'TheFirstCollectionyouwanttocreate',
type:FirstCollection
});
WhichwemaydousingcURL:
echo'{"value":1,"_key":"13"}'|\
curl-d@-http://localhost:8529/_db/_system/collectable_foxx/data/firstCollection/firstCollection
We'dexpectthevaluetobeintherangeof1to5.Maybethesourceofthisdataisaweb-pollorsomethingsimilar.
WenowaddanotherFoxx-routewhichwewanttolinkwithcollectd:
/**
*weuseagroup-byconstructtogetthevalues:
*/
vardb=require('org/arangodb').db;
varsearchQuery='FORxIN@@collectionFILTERx.Date>=@untilcollectvalue=x.valuewithcountintocounterRETURN{[[CONCAT
("choice",value)]:counter}';
controller.get('/firstCollection/lastSeconds/:nSeconds',function(req,res){
varuntil=Date.now()-req.params('nSeconds')*1000;
res.json(
db._query(searchQuery,{
'@collection':FirstCollection_repo.collection.name(),
'until':until
}).toArray()
);
}).pathParam('nSeconds',{
description:'lookuptonSecondsintothepast',
Collectd-MonitoringFoxx
151
type:joi.string().required()
});
Weinspectthereturndocumentusingcurlandjqforniceformatting:
curl'http://localhost:8529/_db/_system/collectable_foxx/data/firstCollection/firstCollection/lastSeconds/10'|jq"."
[
{
"1":3
"3":7
}
]
Wehavetodesignthereturnvaluesinawaythatcollectd'sconfigsyntaxcansimplygrabit.ThisRoutereturnsanobjectwithflatkeyvalueswherekeysmayrangefrom0to5.Wecreateasimplecollectdconfigurationin/etc/collectd/collectd.conf.d/foxx_simple.confthatmatchesourAPI:
#Loadtheplug-in:
LoadPlugincurl_json
#weneedtouseourowntypestogenerateindividualnamesforourgauges:
TypesDB"/etc/collectd/collectd.conf.d/foxx_simple_types.db"
<Plugincurl_json>
#AdjusttheURLsocollectdcanreachyourarangod:
<URL"http://localhost:8529/_db/_system/collectable_foxx/data/firstCollection/firstCollection/lastSeconds/10">
#SetyourauthenticationtoAardvarkhere:
#User"foo"
#Password"bar"
<Key"choice0">
Type"the_values"
</Key>
<Key"choice1">
Type"first_values"
</Key>
<Key"choice2">
Type"second_values"
</Key>
<Key"choice3">
Type"third_values"
</Key>
<Key"choice4">
Type"fourth_values"
</Key>
<Key"choice5">
Type"fifth_values"
</Key>
</URL>
</Plugin>
Togetnicemetricnames,wespecifyourowntypes.dbfilein/etc/collectd/collectd.conf.d/foxx_simple_types.db:
the_valuesvalue:GAUGE:U:U
first_valuesvalue:GAUGE:U:U
second_valuesvalue:GAUGE:U:U
third_valuesvalue:GAUGE:U:U
fourth_valuesvalue:GAUGE:U:U
fifth_valuesvalue:GAUGE:U:U
Author:WilfriedGoesgens
Tags:#monitoring#foxx#json
Collectd-MonitoringFoxx
152