[{"data":1,"prerenderedAt":2559},["ShallowReactive",2],{"docs-nav":3,"docs:\u002Fdocs\u002Fhow-to\u002Fcluster":151},[4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58,61,64,67,70,73,76,79,82,85,88,91,94,97,100,103,106,109,112,115,118,121,124,127,130,133,136,139,142,145,148],{"path":5,"title":6},"\u002Fdocs\u002Faccess-gating","Protect a local route",{"path":8,"title":9},"\u002Fdocs\u002Farchitecture\u002Fsecurity-and-trust","Security and trust",{"path":11,"title":12},"\u002Fdocs\u002Fconcepts\u002Fbuilds-and-deployments","Build and deploy",{"path":14,"title":15},"\u002Fdocs\u002Fconcepts\u002Fconfiguration-ownership","Where configuration belongs",{"path":17,"title":18},"\u002Fdocs\u002Fconcepts\u002Fenvironments-and-secrets","Environment values and secrets",{"path":20,"title":21},"\u002Fdocs\u002Fconcepts\u002Fmental-model","How projects fit together",{"path":23,"title":24},"\u002Fdocs\u002Fconcepts\u002Fplans-and-apply","Plans and apply",{"path":26,"title":27},"\u002Fdocs\u002Fconcepts\u002Ftargets-and-runtimes","Choose a target and runtime",{"path":29,"title":30},"\u002Fdocs\u002Fconfig-files","Ship config files with a service",{"path":32,"title":33},"\u002Fdocs\u002Fcontribute\u002Freleases","Publish a release",{"path":35,"title":36},"\u002Fdocs\u002Fdevelopment-mounts","Development mounts",{"path":38,"title":39},"\u002Fdocs\u002Fgetting-started","Getting started",{"path":41,"title":42},"\u002Fdocs\u002Fhardening","Harden services and choose their runtime",{"path":44,"title":45},"\u002Fdocs\u002Fhelp\u002Ffaq","FAQ",{"path":47,"title":48},"\u002Fdocs\u002Fhelp\u002Ftroubleshooting","Troubleshooting",{"path":50,"title":51},"\u002Fdocs\u002Fhow-to\u002Fcluster","Run a cluster",{"path":53,"title":54},"\u002Fdocs\u002Fhow-to\u002Fdeploy-production","Prepare a deployment environment",{"path":56,"title":57},"\u002Fdocs\u002Fhow-to\u002Ffrom-docker-swarm","Coming from Docker Swarm",{"path":59,"title":60},"\u002Fdocs\u002Fhow-to\u002Fproduction-checklist","Production checklist",{"path":62,"title":63},"\u002Fdocs","mm documentation",{"path":65,"title":66},"\u002Fdocs\u002Fjobs-and-schedules","Jobs and schedules",{"path":68,"title":69},"\u002Fdocs\u002Flocal-dashboard","Local dashboard",{"path":71,"title":72},"\u002Fdocs\u002Flocal-https","Local HTTPS",{"path":74,"title":75},"\u002Fdocs\u002Flocal-services","Local services",{"path":77,"title":78},"\u002Fdocs\u002Fmcp","Let an AI agent work on your project",{"path":80,"title":81},"\u002Fdocs\u002Fmulti-project-hosts","Run multiple projects on one host",{"path":83,"title":84},"\u002Fdocs\u002Fonboarding","Set up a project",{"path":86,"title":87},"\u002Fdocs\u002Fopenbao","Use OpenBao",{"path":89,"title":90},"\u002Fdocs\u002Fproject\u002Fcontributing","Contributing",{"path":92,"title":93},"\u002Fdocs\u002Fproject\u002Flicense","License",{"path":95,"title":96},"\u002Fdocs\u002Fproject\u002Fsecurity","Security",{"path":98,"title":99},"\u002Fdocs\u002Freference\u002Fcapability-matrix","Supported features and platforms",{"path":101,"title":102},"\u002Fdocs\u002Freference\u002Fcli","CLI command reference",{"path":104,"title":105},"\u002Fdocs\u002Freference\u002Fconfiguration","Configuration",{"path":107,"title":108},"\u002Fdocs\u002Freference\u002Fenvironment-variables","Environment variables",{"path":110,"title":111},"\u002Fdocs\u002Freference\u002Ferrors-and-exit-codes","Errors and exit codes",{"path":113,"title":114},"\u002Fdocs\u002Freference\u002Fknown-issues","Known limitations",{"path":116,"title":117},"\u002Fdocs\u002Fremote-access","Manage remote access",{"path":119,"title":120},"\u002Fdocs\u002Fremote-daemon","Set up a remote host",{"path":122,"title":123},"\u002Fdocs\u002Fruntime-observability","Status, logs, and commands",{"path":125,"title":126},"\u002Fdocs\u002Fsecret-delivery","Pass secrets to your app",{"path":128,"title":129},"\u002Fdocs\u002Fsecret-providers","Choose a secret store",{"path":131,"title":132},"\u002Fdocs\u002Fstart\u002Finstall","Install mm",{"path":134,"title":135},"\u002Fdocs\u002Fterminal-ui","Terminal UI",{"path":137,"title":138},"\u002Fdocs\u002Ftutorials\u002Fdotnet-web-api",".NET web API example",{"path":140,"title":141},"\u002Fdocs\u002Ftutorials\u002Ffirst-local-app","First local app",{"path":143,"title":144},"\u002Fdocs\u002Ftutorials\u002Flaravel-rust","Laravel and Rust example",{"path":146,"title":147},"\u002Fdocs\u002Ftutorials\u002Fnode-fullstack","Node full stack example",{"path":149,"title":150},"\u002Fdocs\u002Fvault","Use HashiCorp Vault",{"id":152,"title":51,"body":153,"description":2553,"extension":2554,"meta":2555,"navigation":1141,"path":50,"seo":2556,"stem":2557,"__hash__":2558},"docs\u002Fdocs\u002Fhow-to\u002Fcluster.md",{"type":154,"value":155,"toc":2533},"minimark",[156,169,185,190,273,277,280,435,446,458,483,491,502,506,509,597,634,637,655,661,676,680,690,929,960,964,973,1009,1012,1018,1021,1027,1046,1056,1060,1063,1069,1072,1090,1097,1104,1108,1115,1251,1271,1331,1337,1348,1352,1359,1407,1435,1446,1449,1470,1476,1479,1522,1529,1533,1536,1605,1611,1617,1632,1638,1641,1650,1656,1675,1682,1686,1695,1710,1716,1722,1725,1788,1815,1819,1826,1901,1915,1919,1929,2033,2036,2078,2082,2088,2227,2247,2251,2257,2310,2320,2324,2330,2358,2367,2371,2386,2469,2477,2481,2510,2514,2522,2529],[157,158,159,160,164,165,168],"p",{},"A cluster is a target that lists several hosts under ",[161,162,163],"code",{},"nodes:",". ",[161,166,167],{},"mm"," sets every\nhost up, joins them over a private network, and deploys your Compose project\nacross them one node at a time. There are no managers, join tokens, or quorum\nto look after: a cluster has nodes, services, networks, and volumes.",[157,170,171,172,176,177,179,180,184],{},"This guide goes from a node list to a running deployment, then through the\ntasks that follow. Each section links to the full description in\n",[173,174,120],"a",{"href":175},"..\u002Fremote-daemon",", which covers everything ",[161,178,167],{}," puts\non a node. If you run Docker Swarm today, ",[173,181,183],{"href":182},".\u002Ffrom-docker-swarm","Coming from Docker\nSwarm"," maps its concepts and Compose keys to these.",[186,187,189],"h2",{"id":188},"what-you-need","What you need",[191,192,193,209,212,231,256],"ul",{},[194,195,196,197,199,200,203,204,208],"li",{},"A project that ",[161,198,167],{}," already deploys: ",[161,201,202],{},".config\u002Fmm.yaml"," and its Compose files\nin a git repository. To create one, ",[173,205,207],{"href":206},"..\u002Fonboarding","add mm to a project",".",[194,210,211],{},"Two or more Debian or Ubuntu hosts with systemd and OpenSSH. Your machine\nmust reach each one over SSH, and the hosts must reach each other on UDP\nport 51820.",[194,213,214,215,218,219,222,223,226,227,230],{},"This device's mm key (",[161,216,217],{},"mm key create",", then ",[161,220,221],{},"mm key show",") authorized for\n",[161,224,225],{},"root"," on every host. Setup logs in with that key, never with ",[161,228,229],{},"~\u002F.ssh"," or an\nSSH agent.",[194,232,233,234,237,238,241,242,244,245,248,249,251,252,255],{},"On your machine: ",[161,235,236],{},"ansible-playbook",", ",[161,239,240],{},"ssh-keyscan",", and Linux builds of ",[161,243,167],{},"\nand ",[161,246,247],{},"mmd"," from the same release. Installing ",[161,250,167],{}," on Linux with\n",[161,253,254],{},"--with-daemon"," provides both.",[194,257,258,259,265,266,268,269,208],{},"A git origin that every node can fetch the project from, or\n",[173,260,262],{"href":261},"#deploy-without-git-on-the-nodes",[161,263,264],{},"delivery: push"," on the target so that\n",[161,267,167],{}," sends each commit itself. Also a break-glass key pair whose private\nhalf you keep offline. See the ",[173,270,272],{"href":271},"..\u002Fremote-daemon#requirements","requirements",[186,274,276],{"id":275},"declare-the-nodes","Declare the nodes",[157,278,279],{},"Give the target a name and a host for each node, and point an environment at\nit:",[281,282,287],"pre",{"className":283,"code":284,"language":285,"meta":286,"style":286},"language-yaml shiki shiki-themes mm","# .config\u002Fmm.yaml\ntargets:\n  production:\n    nodes:\n      - {name: app-1, host: 203.0.113.11}\n      - {name: app-2, host: 203.0.113.12}\n      - {name: db-1, host: 203.0.113.13}\nenvironments:\n  production:\n    target: production\n","yaml","",[161,288,289,298,309,317,325,359,384,409,417,424],{"__ignoreMap":286},[290,291,294],"span",{"class":292,"line":293},"line",1,[290,295,297],{"class":296},"sP7nD","# .config\u002Fmm.yaml\n",[290,299,301,305],{"class":292,"line":300},2,[290,302,304],{"class":303},"sWZmh","targets",[290,306,308],{"class":307},"sIbZ0",":\n",[290,310,312,315],{"class":292,"line":311},3,[290,313,314],{"class":303},"  production",[290,316,308],{"class":307},[290,318,320,323],{"class":292,"line":319},4,[290,321,322],{"class":303},"    nodes",[290,324,308],{"class":307},[290,326,328,331,334,337,340,344,347,350,352,356],{"class":292,"line":327},5,[290,329,330],{"class":307},"      -",[290,332,333],{"class":307}," {",[290,335,336],{"class":303},"name",[290,338,339],{"class":307},":",[290,341,343],{"class":342},"sv8ni"," app-1",[290,345,346],{"class":307},",",[290,348,349],{"class":303}," host",[290,351,339],{"class":307},[290,353,355],{"class":354},"s1Wbj"," 203.0.113.11",[290,357,358],{"class":307},"}\n",[290,360,362,364,366,368,370,373,375,377,379,382],{"class":292,"line":361},6,[290,363,330],{"class":307},[290,365,333],{"class":307},[290,367,336],{"class":303},[290,369,339],{"class":307},[290,371,372],{"class":342}," app-2",[290,374,346],{"class":307},[290,376,349],{"class":303},[290,378,339],{"class":307},[290,380,381],{"class":354}," 203.0.113.12",[290,383,358],{"class":307},[290,385,387,389,391,393,395,398,400,402,404,407],{"class":292,"line":386},7,[290,388,330],{"class":307},[290,390,333],{"class":307},[290,392,336],{"class":303},[290,394,339],{"class":307},[290,396,397],{"class":342}," db-1",[290,399,346],{"class":307},[290,401,349],{"class":303},[290,403,339],{"class":307},[290,405,406],{"class":354}," 203.0.113.13",[290,408,358],{"class":307},[290,410,412,415],{"class":292,"line":411},8,[290,413,414],{"class":303},"environments",[290,416,308],{"class":307},[290,418,420,422],{"class":292,"line":419},9,[290,421,314],{"class":303},[290,423,308],{"class":307},[290,425,427,430,432],{"class":292,"line":426},10,[290,428,429],{"class":303},"    target",[290,431,339],{"class":307},[290,433,434],{"class":342}," production\n",[157,436,437,438,441,442,208],{},"That is a complete cluster target. The mesh subnet, each node's mesh address,\nthe WireGuard port, the address range for networks, and the node that serves\nshared volumes all have defaults; the optional ",[161,439,440],{},"cluster:"," block exists only to\n",[173,443,445],{"href":444},"..\u002Fremote-daemon#change-the-defaults","change one",[157,447,448,449,452,453,457],{},"Setup also needs to know where the nodes fetch the project from and which keys\ncan always get in. Add a ",[161,450,451],{},"setup:"," block to the target as described in\n",[173,454,456],{"href":455},"..\u002Fremote-daemon#declare-what-setup-installs","declare what setup installs",",\nthen check the file:",[281,459,463],{"className":460,"code":461,"language":462,"meta":286,"style":286},"language-sh shiki shiki-themes mm","mm -e production config validate\n","sh",[161,464,465],{"__ignoreMap":286},[290,466,467,470,474,477,480],{"class":292,"line":293},[290,468,167],{"class":469},"svrjQ",[290,471,473],{"class":472},"sbRf1"," -e",[290,475,476],{"class":342}," production",[290,478,479],{"class":342}," config",[290,481,482],{"class":342}," validate\n",[281,484,489],{"className":485,"code":487,"language":488,"meta":286},[486],"language-text","shop [production]\n  config: \u002Fsrv\u002Fcode\u002Fshop\u002F.config\u002Fmm.yaml\n  config:      valid\n","text",[161,490,487],{"__ignoreMap":286},[157,492,493,494,497,498,501],{},"A node name that is not a hostname label, a duplicate host, a ",[161,495,496],{},"placement"," that\nnames an unknown node, and a Compose service with no entry under ",[161,499,500],{},"modules"," are\nreported instead, each with the key to change.",[186,503,505],{"id":504},"set-up-the-nodes","Set up the nodes",[157,507,508],{},"Setup is a plan you review, then an apply that runs the same steps on every\nnode:",[281,510,512],{"className":460,"code":511,"language":462,"meta":286,"style":286},"mm -e production target setup deploy-key\nmm config split targets\nmm -e production target setup plan\nmm -e production target setup apply --plan PLAN_HASH --yes\nmm -e production target setup record\n",[161,513,514,531,543,558,582],{"__ignoreMap":286},[290,515,516,518,520,522,525,528],{"class":292,"line":293},[290,517,167],{"class":469},[290,519,473],{"class":472},[290,521,476],{"class":342},[290,523,524],{"class":342}," target",[290,526,527],{"class":342}," setup",[290,529,530],{"class":342}," deploy-key\n",[290,532,533,535,537,540],{"class":292,"line":300},[290,534,167],{"class":469},[290,536,479],{"class":342},[290,538,539],{"class":342}," split",[290,541,542],{"class":342}," targets\n",[290,544,545,547,549,551,553,555],{"class":292,"line":311},[290,546,167],{"class":469},[290,548,473],{"class":472},[290,550,476],{"class":342},[290,552,524],{"class":342},[290,554,527],{"class":342},[290,556,557],{"class":342}," plan\n",[290,559,560,562,564,566,568,570,573,576,579],{"class":292,"line":319},[290,561,167],{"class":469},[290,563,473],{"class":472},[290,565,476],{"class":342},[290,567,524],{"class":342},[290,569,527],{"class":342},[290,571,572],{"class":342}," apply",[290,574,575],{"class":472}," --plan",[290,577,578],{"class":342}," PLAN_HASH",[290,580,581],{"class":472}," --yes\n",[290,583,584,586,588,590,592,594],{"class":292,"line":327},[290,585,167],{"class":469},[290,587,473],{"class":472},[290,589,476],{"class":342},[290,591,524],{"class":342},[290,593,527],{"class":342},[290,595,596],{"class":342}," record\n",[157,598,599,602,603,164,606,609,610,613,614,617,618,620,621,624,625,628,629,633],{},[161,600,601],{},"deploy-key"," prints a public key once; add it to the git origin as a read-only\ndeploy key and set it as ",[161,604,605],{},"setup.deploy-key.public-key",[161,607,608],{},"plan"," scans every\nnode's SSH host key and shows what each node gets, including its mesh peers.\n",[161,611,612],{},"apply"," installs Docker Engine with the Compose plugin (or Podman, for an\nenvironment that uses ",[161,615,616],{},"podman-compose",") and ",[161,619,247],{},", brings up the\nWireGuard mesh, prepares shared storage, pushes the environment's secrets, and\napplies the users file. ",[161,622,623],{},"record"," writes each node's address, host key, and\ndaemon identity to ",[161,626,627],{},".config\u002Fmm\u002Ftargets.yaml","; commit and push that file.\n",[173,630,632],{"href":631},"..\u002Fremote-daemon#run-setup","Run setup"," explains each step and what to do\nwhen one node cannot be reached.",[157,635,636],{},"Check the result:",[281,638,640],{"className":460,"code":639,"language":462,"meta":286,"style":286},"mm -e production target nodes\n",[161,641,642],{"__ignoreMap":286},[290,643,644,646,648,650,652],{"class":292,"line":293},[290,645,167],{"class":469},[290,647,473],{"class":472},[290,649,476],{"class":342},[290,651,524],{"class":342},[290,653,654],{"class":342}," nodes\n",[281,656,659],{"className":657,"code":658,"language":488,"meta":286},[486],"Target production: 3 node(s)\n  NODE             ADDRESS                  MESH            STATE        MMD        DEPLOYED\n  app-1            203.0.113.11:22          10.90.0.1       ready        0.1.0      nothing yet\n  app-2            203.0.113.12:22          10.90.0.2       ready        0.1.0      nothing yet\n  db-1             203.0.113.13:22          10.90.0.3       ready        0.1.0      nothing yet\n\nNetworks (cluster.networks 10.64.0.0\u002F12)\n  No network of this environment has addresses yet; the first deployment gives them out.\n",[161,660,658],{"__ignoreMap":286},[157,662,663,664,667,668,671,672,208],{},"Every node is ",[161,665,666],{},"ready",". Below this the command lists the addresses each node\nalready uses, which the cluster's ranges must not overlap.\n",[161,669,670],{},"mm -e production target mesh status"," shows the WireGuard link between each\npair of nodes and names the port to open when two of them have no handshake;\nsee ",[173,673,675],{"href":674},"..\u002Fremote-daemon#the-mesh-between-the-nodes","the mesh between the nodes",[186,677,679],{"id":678},"say-where-each-service-runs","Say where each service runs",[157,681,682,683,685,686,689],{},"Give a service a ",[161,684,496],{},": a node name, a group, the word ",[161,687,688],{},"all",", or a list.",[281,691,693],{"className":283,"code":692,"language":285,"meta":286,"style":286},"targets:\n  production:\n    nodes:\n      - {name: app-1, host: 203.0.113.11}\n      - {name: app-2, host: 203.0.113.12}\n      - {name: db-1, host: 203.0.113.13}\n    cluster:\n      groups: {web: [app-1, app-2]}\nmodules:\n  shop:\n    path: shop\n    services:\n      web: {stack: app, placement: web} # app-1 and app-2\n      db: {stack: app, placement: db-1} # db-1 only\n      worker: {stack: app} # no placement: the first node, app-1\n",[161,694,695,701,707,713,735,757,779,786,813,819,826,837,845,879,908],{"__ignoreMap":286},[290,696,697,699],{"class":292,"line":293},[290,698,304],{"class":303},[290,700,308],{"class":307},[290,702,703,705],{"class":292,"line":300},[290,704,314],{"class":303},[290,706,308],{"class":307},[290,708,709,711],{"class":292,"line":311},[290,710,322],{"class":303},[290,712,308],{"class":307},[290,714,715,717,719,721,723,725,727,729,731,733],{"class":292,"line":319},[290,716,330],{"class":307},[290,718,333],{"class":307},[290,720,336],{"class":303},[290,722,339],{"class":307},[290,724,343],{"class":342},[290,726,346],{"class":307},[290,728,349],{"class":303},[290,730,339],{"class":307},[290,732,355],{"class":354},[290,734,358],{"class":307},[290,736,737,739,741,743,745,747,749,751,753,755],{"class":292,"line":327},[290,738,330],{"class":307},[290,740,333],{"class":307},[290,742,336],{"class":303},[290,744,339],{"class":307},[290,746,372],{"class":342},[290,748,346],{"class":307},[290,750,349],{"class":303},[290,752,339],{"class":307},[290,754,381],{"class":354},[290,756,358],{"class":307},[290,758,759,761,763,765,767,769,771,773,775,777],{"class":292,"line":361},[290,760,330],{"class":307},[290,762,333],{"class":307},[290,764,336],{"class":303},[290,766,339],{"class":307},[290,768,397],{"class":342},[290,770,346],{"class":307},[290,772,349],{"class":303},[290,774,339],{"class":307},[290,776,406],{"class":354},[290,778,358],{"class":307},[290,780,781,784],{"class":292,"line":386},[290,782,783],{"class":303},"    cluster",[290,785,308],{"class":307},[290,787,788,791,793,795,798,800,803,806,808,810],{"class":292,"line":411},[290,789,790],{"class":303},"      groups",[290,792,339],{"class":307},[290,794,333],{"class":307},[290,796,797],{"class":303},"web",[290,799,339],{"class":307},[290,801,802],{"class":307}," [",[290,804,805],{"class":342},"app-1",[290,807,346],{"class":307},[290,809,372],{"class":342},[290,811,812],{"class":307},"]}\n",[290,814,815,817],{"class":292,"line":419},[290,816,500],{"class":303},[290,818,308],{"class":307},[290,820,821,824],{"class":292,"line":426},[290,822,823],{"class":303},"  shop",[290,825,308],{"class":307},[290,827,829,832,834],{"class":292,"line":828},11,[290,830,831],{"class":303},"    path",[290,833,339],{"class":307},[290,835,836],{"class":342}," shop\n",[290,838,840,843],{"class":292,"line":839},12,[290,841,842],{"class":303},"    services",[290,844,308],{"class":307},[290,846,848,851,853,855,858,860,863,865,868,870,873,876],{"class":292,"line":847},13,[290,849,850],{"class":303},"      web",[290,852,339],{"class":307},[290,854,333],{"class":307},[290,856,857],{"class":303},"stack",[290,859,339],{"class":307},[290,861,862],{"class":342}," app",[290,864,346],{"class":307},[290,866,867],{"class":303}," placement",[290,869,339],{"class":307},[290,871,872],{"class":342}," web",[290,874,875],{"class":307},"}",[290,877,878],{"class":296}," # app-1 and app-2\n",[290,880,882,885,887,889,891,893,895,897,899,901,903,905],{"class":292,"line":881},14,[290,883,884],{"class":303},"      db",[290,886,339],{"class":307},[290,888,333],{"class":307},[290,890,857],{"class":303},[290,892,339],{"class":307},[290,894,862],{"class":342},[290,896,346],{"class":307},[290,898,867],{"class":303},[290,900,339],{"class":307},[290,902,397],{"class":342},[290,904,875],{"class":307},[290,906,907],{"class":296}," # db-1 only\n",[290,909,911,914,916,918,920,922,924,926],{"class":292,"line":910},15,[290,912,913],{"class":303},"      worker",[290,915,339],{"class":307},[290,917,333],{"class":307},[290,919,857],{"class":303},[290,921,339],{"class":307},[290,923,862],{"class":342},[290,925,875],{"class":307},[290,927,928],{"class":296}," # no placement: the first node, app-1\n",[157,930,931,932,935,936,939,940,944,945,947,948,950,951,954,955,959],{},"A service runs one container on each node its placement names, unless it sets\n",[161,933,934],{},"replicas"," or ",[161,937,938],{},"autoscale"," (see ",[173,941,943],{"href":942},"#scale-a-service","scale a service","). Every service\nin a stack's Compose files needs an entry under ",[161,946,500],{},", so that ",[161,949,167],{}," knows\nwhere it runs; ",[161,952,953],{},"mm config validate"," names any that is missing. A named Compose\nvolume is local to its node, so keep a service that stores data pinned to one\nnode. ",[173,956,958],{"href":957},"..\u002Fremote-daemon#place-services-on-nodes","Place services on nodes","\ncovers moving a service and the Compose keys that tie two services to one\nhost.",[186,961,963],{"id":962},"deploy","Deploy",[157,965,966,967,969,970,339],{},"Push the commit, then plan and apply it. Every node fetches the commit from\nthe origin itself, or receives it from ",[161,968,167],{}," with\n",[173,971,972],{"href":261},"push delivery",[281,974,976],{"className":460,"code":975,"language":462,"meta":286,"style":286},"mm -e production deploy plan\nmm -e production deploy apply --plan PLAN_HASH --yes\n",[161,977,978,991],{"__ignoreMap":286},[290,979,980,982,984,986,989],{"class":292,"line":293},[290,981,167],{"class":469},[290,983,473],{"class":472},[290,985,476],{"class":342},[290,987,988],{"class":342}," deploy",[290,990,557],{"class":342},[290,992,993,995,997,999,1001,1003,1005,1007],{"class":292,"line":300},[290,994,167],{"class":469},[290,996,473],{"class":472},[290,998,476],{"class":342},[290,1000,988],{"class":342},[290,1002,572],{"class":342},[290,1004,575],{"class":472},[290,1006,578],{"class":342},[290,1008,581],{"class":472},[157,1010,1011],{},"The plan opens with what the deployment does across the nodes, then has one\nsection per node: what each node starts, changes, and stops, and the addresses\neach network gets there. One hash covers every section:",[281,1013,1016],{"className":1014,"code":1015,"language":488,"meta":286},[486],"Cluster deployment plan 0a4ba0ae34677ecf6507139bc887347d251e8ca7ba19e9f006fc2f016be9f9ab\n  target:   production (app-1, app-2, db-1)\n  commit:   2e445474cb8dc3608f384f908f60ab7cf47c89e9\n  expires:  2026-11-13T17:43:32Z\n  networks span the nodes: addresses from 10.64.0.0\u002F12, between nodes through mm0 (MTU 1420)\n  service names: each node answers its own containers, on its address on each network (port 53), with a service's healthy containers on the same node, or else on every node\n  rollout: app on app-1, then app-2, then db-1\n           a stack may take 10m on a node to become healthy (deploy.convergence-timeout); the next node starts once the others resolve its containers\n           on failure every node this deployment changed returns to what it ran before\n",[161,1017,1015],{"__ignoreMap":286},[157,1019,1020],{},"Apply replaces one node at a time. A node is done when its containers pass\ntheir health checks and every other node resolves them by name:",[281,1022,1025],{"className":1023,"code":1024,"language":488,"meta":286},[486],"==> app-1: preparing\n==> app-1: app\n==> app-2: preparing\n==> app-2: app\n==> db-1: preparing\n==> db-1: app\n==> app-1: finishing\n==> app-2: finishing\n==> db-1: finishing\nDeployment 0a4ba0ae34677ecf6507139bc887347d251e8ca7ba19e9f006fc2f016be9f9ab on production [production]\n  app-1            applied      stacks: app; 0 route(s)\n  app-2            applied      stacks: app; 0 route(s)\n  db-1             applied      stacks: app; 0 route(s)\n",[161,1026,1024],{"__ignoreMap":286},[157,1028,1029,1030,1033,1034,1037,1038,1041,1042,208],{},"Give each service a health check in its Compose file: without one, a container\ncounts as healthy as soon as it runs. To replace a container without a gap,\nset ",[161,1031,1032],{},"update: {order: start-first}"," on the service, and the apply says\n",[161,1035,1036],{},"app: starting the new web next to the old"," on each node. To order stacks, set\n",[161,1039,1040],{},"after:"," on a stack. Both are in ",[173,1043,1045],{"href":1044},"..\u002Fremote-daemon#rolling-deploys-and-rollback","rolling deploys and\nrollback",[157,1047,1048,1049,1051,1052,208],{},"Modules that ",[161,1050,167],{}," builds deploy from a build receipt, because a node never\nbuilds. Follow ",[173,1053,1055],{"href":1054},"..\u002Fconcepts\u002Fbuilds-and-deployments#deploy-a-build-to-a-remote-host","deploy a build to a remote\nhost",[186,1057,1059],{"id":1058},"roll-back","Roll back",[157,1061,1062],{},"A deployment that fails rolls itself back. When a container fails its health\ncheck, exits, or keeps restarting, the deployment stops, every node it changed\nreturns to what it ran before, and the nodes it had not reached are never\ntouched:",[281,1064,1067],{"className":1065,"code":1066,"language":488,"meta":286},[486],"==> app-1: preparing\n==> app-1: app\n==> app-2: preparing\n==> app-2: app\n==> app-2: rolling back\n==> app-1: rolling back\nDeployment 85549657b9e8dbab7d79adda5e1cbeb919803cfcab243882043abfbae414ebb4 on production [production]\n  app-1            rolled back  runs what it ran before\n  app-2            FAILED       service 'web' of stack 'app': its health check fails; rolled back to what it ran before\n                   last log lines of the failed container:\n                     listening on :3000\n                     cannot reach the database at db:5432\n  db-1             not started  unchanged\n",[161,1068,1066],{"__ignoreMap":286},[157,1070,1071],{},"To return from a deployment that finished:",[281,1073,1075],{"className":460,"code":1074,"language":462,"meta":286,"style":286},"mm -e production deploy rollback\n",[161,1076,1077],{"__ignoreMap":286},[290,1078,1079,1081,1083,1085,1087],{"class":292,"line":293},[290,1080,167],{"class":469},[290,1082,473],{"class":472},[290,1084,476],{"class":342},[290,1086,988],{"class":342},[290,1088,1089],{"class":342}," rollback\n",[157,1091,1092,1093,1096],{},"It prints a plan for the commit the nodes ran before and applies it after you\nconfirm. ",[161,1094,1095],{},"mm -e production deploy status"," shows what every node runs, what it\nran before, and a rollout that was interrupted.",[157,1098,1099,1100,208],{},"Volumes are not rolled back: the earlier version runs against the data the\nlater one left. Plan schema changes so that the previous version still works\nwith them. More in ",[173,1101,1103],{"href":1102},"..\u002Fremote-daemon#return-to-the-previous-deployment","return to the previous\ndeployment",[186,1105,1107],{"id":1106},"connect-services-across-nodes","Connect services across nodes",[157,1109,1110,1111,1114],{},"Declare a network in ",[161,1112,1113],{},"mm.yaml"," and put services on it. It spans every node\nthat runs one of them:",[281,1116,1118],{"className":283,"code":1117,"language":285,"meta":286,"style":286},"networks:\n  backend: {}\n\nmodules:\n  shop:\n    services:\n      web: {stack: app, role: http, networks: [backend], placement: web}\n      db: {stack: app, role: database, networks: [backend], placement: db-1}\n",[161,1119,1120,1127,1137,1143,1149,1155,1161,1208],{"__ignoreMap":286},[290,1121,1122,1125],{"class":292,"line":293},[290,1123,1124],{"class":303},"networks",[290,1126,308],{"class":307},[290,1128,1129,1132,1134],{"class":292,"line":300},[290,1130,1131],{"class":303},"  backend",[290,1133,339],{"class":307},[290,1135,1136],{"class":307}," {}\n",[290,1138,1139],{"class":292,"line":311},[290,1140,1142],{"emptyLinePlaceholder":1141},true,"\n",[290,1144,1145,1147],{"class":292,"line":319},[290,1146,500],{"class":303},[290,1148,308],{"class":307},[290,1150,1151,1153],{"class":292,"line":327},[290,1152,823],{"class":303},[290,1154,308],{"class":307},[290,1156,1157,1159],{"class":292,"line":361},[290,1158,842],{"class":303},[290,1160,308],{"class":307},[290,1162,1163,1165,1167,1169,1171,1173,1175,1177,1180,1182,1185,1187,1190,1192,1194,1197,1200,1202,1204,1206],{"class":292,"line":386},[290,1164,850],{"class":303},[290,1166,339],{"class":307},[290,1168,333],{"class":307},[290,1170,857],{"class":303},[290,1172,339],{"class":307},[290,1174,862],{"class":342},[290,1176,346],{"class":307},[290,1178,1179],{"class":303}," role",[290,1181,339],{"class":307},[290,1183,1184],{"class":342}," http",[290,1186,346],{"class":307},[290,1188,1189],{"class":303}," networks",[290,1191,339],{"class":307},[290,1193,802],{"class":307},[290,1195,1196],{"class":342},"backend",[290,1198,1199],{"class":307},"],",[290,1201,867],{"class":303},[290,1203,339],{"class":307},[290,1205,872],{"class":342},[290,1207,358],{"class":307},[290,1209,1210,1212,1214,1216,1218,1220,1222,1224,1226,1228,1231,1233,1235,1237,1239,1241,1243,1245,1247,1249],{"class":292,"line":411},[290,1211,884],{"class":303},[290,1213,339],{"class":307},[290,1215,333],{"class":307},[290,1217,857],{"class":303},[290,1219,339],{"class":307},[290,1221,862],{"class":342},[290,1223,346],{"class":307},[290,1225,1179],{"class":303},[290,1227,339],{"class":307},[290,1229,1230],{"class":342}," database",[290,1232,346],{"class":307},[290,1234,1189],{"class":303},[290,1236,339],{"class":307},[290,1238,802],{"class":307},[290,1240,1196],{"class":342},[290,1242,1199],{"class":307},[290,1244,867],{"class":303},[290,1246,339],{"class":307},[290,1248,397],{"class":342},[290,1250,358],{"class":307},[157,1252,1253,1255,1256,1258,1259,1262,1263,1266,1267,1270],{},[161,1254,797],{}," on ",[161,1257,805],{}," reaches the database at ",[161,1260,1261],{},"db:5432",", although ",[161,1264,1265],{},"db"," runs on\n",[161,1268,1269],{},"db-1",". The traffic travels inside the mesh, encrypted, and two containers\nthat share no network cannot reach each other.",[1272,1273,1274,1287],"table",{},[1275,1276,1277],"thead",{},[1278,1279,1280,1284],"tr",{},[1281,1282,1283],"th",{},"Name",[1281,1285,1286],{},"Resolves to",[1288,1289,1290,1303,1316],"tbody",{},[1278,1291,1292,1297],{},[1293,1294,1295],"td",{},[161,1296,1265],{},[1293,1298,1299,1300,1302],{},"The healthy containers of ",[161,1301,1265],{}," on the asking node, or on every node when it has none.",[1278,1304,1305,1310],{},[1293,1306,1307],{},[161,1308,1309],{},"tasks.db",[1293,1311,1312,1313,1315],{},"Every healthy container of ",[161,1314,1265],{},", on every node.",[1278,1317,1318,1323],{},[1293,1319,1320],{},[161,1321,1322],{},"db.db-1",[1293,1324,1325,1326,1328,1329,208],{},"The healthy container of ",[161,1327,1265],{}," on node ",[161,1330,1269],{},[157,1332,1333,1336],{},[161,1334,1335],{},"mm -e production service status"," lists what each name resolves to. A network\nthat only a Compose file declares stays local to its host.",[157,1338,1339,1340,244,1344,208],{},"Limits: a network spans at most 16 nodes, a node runs at most 253 containers\nof one network, and the default address range holds 256 networks. Addresses\nare IPv4. See ",[173,1341,1343],{"href":1342},"..\u002Fremote-daemon#networks-across-nodes","networks across nodes",[173,1345,1347],{"href":1346},"..\u002Fremote-daemon#service-names-across-nodes","service names across nodes",[186,1349,1351],{"id":1350},"share-a-volume-between-nodes","Share a volume between nodes",[157,1353,1354,1355,1358],{},"Use a shared volume for files that the containers of a service all need on\nevery node, such as uploads or generated assets. List the Compose volume under\n",[161,1356,1357],{},"shared-volumes"," on its stack:",[281,1360,1362],{"className":283,"code":1361,"language":285,"meta":286,"style":286},"stacks:\n  app:\n    files: [compose.yaml]\n    shared-volumes: [uploads]\n",[161,1363,1364,1371,1378,1393],{"__ignoreMap":286},[290,1365,1366,1369],{"class":292,"line":293},[290,1367,1368],{"class":303},"stacks",[290,1370,308],{"class":307},[290,1372,1373,1376],{"class":292,"line":300},[290,1374,1375],{"class":303},"  app",[290,1377,308],{"class":307},[290,1379,1380,1383,1385,1387,1390],{"class":292,"line":311},[290,1381,1382],{"class":303},"    files",[290,1384,339],{"class":307},[290,1386,802],{"class":307},[290,1388,1389],{"class":342},"compose.yaml",[290,1391,1392],{"class":307},"]\n",[290,1394,1395,1398,1400,1402,1405],{"class":292,"line":319},[290,1396,1397],{"class":303},"    shared-volumes",[290,1399,339],{"class":307},[290,1401,802],{"class":307},[290,1403,1404],{"class":342},"uploads",[290,1406,1392],{"class":307},[157,1408,1409,1413,1414,164,1416,1418,1419,935,1422,1425,1426,1429,1430,1434],{},[1410,1411,1412],"strong",{},"Use a local volume for anything where performance matters: databases,\nqueues, search indexes, caches."," A shared volume is a network mount. Next to\nthe storage node, creating a file takes 7 to 9 ms instead of 0.15 ms; on a\nnode 58 ms away, every file opened and every write flushed waits a round trip,\nand a large read runs at 8 MB\u002Fs instead of 1 GB\u002Fs. It also does not give the\nlocking guarantees a database needs, and it stops answering while the node\nthat keeps it is down. Give such a service an ordinary volume and pin it to one\nnode with ",[161,1415,496],{},[161,1417,953],{}," refuses a service with\n",[161,1420,1421],{},"role: database",[161,1423,1424],{},"role: queue"," on a shared volume; ",[161,1427,1428],{},"allow-shared-volumes: true"," on the service overrides that. The ",[173,1431,1433],{"href":1432},"..\u002Fremote-daemon#shared-volumes-across-nodes","measurements","\nare in the remote host guide.",[157,1436,1437,1438,1441,1442,1445],{},"The Compose file stays as it is. One node, the first unless you set\n",[161,1439,1440],{},"cluster.storage",", keeps the files under ",[161,1443,1444],{},"\u002Fvar\u002Flib\u002Fmm\u002Fstorage"," and serves them\nto the other nodes over the mesh with NFS. Nothing is installed on the nodes\nthat mount it. The deployment plan says which node keeps each volume and which\nnodes mount it, and the deployment creates the directory before it replaces a\ncontainer.",[157,1447,1448],{},"Check it:",[281,1450,1452],{"className":460,"code":1451,"language":462,"meta":286,"style":286},"mm -e production target storage status\n",[161,1453,1454],{"__ignoreMap":286},[290,1455,1456,1458,1460,1462,1464,1467],{"class":292,"line":293},[290,1457,167],{"class":469},[290,1459,473],{"class":472},[290,1461,476],{"class":342},[290,1463,524],{"class":342},[290,1465,1466],{"class":342}," storage",[290,1468,1469],{"class":342}," status\n",[281,1471,1474],{"className":1472,"code":1473,"language":488,"meta":286},[486],"Target production: app-1 keeps shared volumes in \u002Fvar\u002Flib\u002Fmm\u002Fstorage, 26.2 GiB free of 29.3 GiB\n\n  app\u002Fuploads              12.0 KiB in 3 files; mounted on app-1, app-2\n\n  app-1                    keeps the volumes and serves them on 10.90.0.1\n  app-2                    reaches app-1 in under 1 ms\n  db-1                     reaches app-1 in 58 ms\n\nEvery node reaches the storage node.\n",[161,1475,1473],{"__ignoreMap":286},[157,1477,1478],{},"What to know before you rely on it:",[191,1480,1481,1490,1493,1503,1506,1512],{},[194,1482,1483,1484,1486,1487,1489],{},"The storage node is a single point of failure. While it is down, reads and\nwrites of shared files on the other nodes wait, and continue when it is\nback. ",[161,1485,167],{}," does not replicate or back up the files: back up\n",[161,1488,1444],{}," on the storage node yourself.",[194,1491,1492],{},"A node that boots while the storage node is away starts its other services;\nthe ones that mount a shared volume start once the storage node answers.",[194,1494,1495,1496,1498,1499,1502],{},"Taking a volume out of ",[161,1497,1357],{},", a rollback, and a failed deployment\nnever delete files. ",[161,1500,1501],{},"mm -e production target storage delete uploads"," shows\nwhat it would remove and deletes it only with the hash it prints.",[194,1504,1505],{},"A volume that becomes shared starts empty on the storage node. Copy the\nfiles from the old local volume yourself.",[194,1507,1508,1509,1511],{},"Changing ",[161,1510,1440],{}," is refused while the old storage node holds\nfiles, with the steps to move them.",[194,1513,1514,1515,1517,1518,1521],{},"Shared volumes need the mesh ",[161,1516,167],{}," builds. A target that sets\n",[161,1519,1520],{},"cluster.mesh.interface"," serves none.",[157,1523,1524,1525,1528],{},"See ",[173,1526,1527],{"href":1432},"shared volumes across nodes","\nfor where the files are, what happens while the storage node is down, and how\nthe export is kept private.",[186,1530,1532],{"id":1531},"read-logs-and-status","Read logs and status",[157,1534,1535],{},"A service is all of its containers on all of its nodes, and every command\nmeans all of them unless you narrow it:",[281,1537,1539],{"className":460,"code":1538,"language":462,"meta":286,"style":286},"mm -e production service status\nmm -e production service logs web --follow\nmm -e production service logs web --node app-2\nmm -e production service logs\n",[161,1540,1541,1554,1572,1592],{"__ignoreMap":286},[290,1542,1543,1545,1547,1549,1552],{"class":292,"line":293},[290,1544,167],{"class":469},[290,1546,473],{"class":472},[290,1548,476],{"class":342},[290,1550,1551],{"class":342}," service",[290,1553,1469],{"class":342},[290,1555,1556,1558,1560,1562,1564,1567,1569],{"class":292,"line":300},[290,1557,167],{"class":469},[290,1559,473],{"class":472},[290,1561,476],{"class":342},[290,1563,1551],{"class":342},[290,1565,1566],{"class":342}," logs",[290,1568,872],{"class":342},[290,1570,1571],{"class":472}," --follow\n",[290,1573,1574,1576,1578,1580,1582,1584,1586,1589],{"class":292,"line":311},[290,1575,167],{"class":469},[290,1577,473],{"class":472},[290,1579,476],{"class":342},[290,1581,1551],{"class":342},[290,1583,1566],{"class":342},[290,1585,872],{"class":342},[290,1587,1588],{"class":472}," --node",[290,1590,1591],{"class":342}," app-2\n",[290,1593,1594,1596,1598,1600,1602],{"class":292,"line":319},[290,1595,167],{"class":469},[290,1597,473],{"class":472},[290,1599,476],{"class":342},[290,1601,1551],{"class":342},[290,1603,1604],{"class":342}," logs\n",[157,1606,1607,1610],{},[161,1608,1609],{},"service status"," asks every node at once and lists each service once, with a\nrow for each node's container:",[281,1612,1615],{"className":1613,"code":1614,"language":488,"meta":286},[486],"  STACK  SERVICE  STATE         NODE   CONTAINER  HEALTH   STATUS      IMAGE\n  app    web      degraded 1\u002F2  app-1  web-1      healthy  Up 2 hours  registry.example.com\u002Fshop\u002Fweb:1.4.2\n                                app-2  -          -        no answer   -\n  app    db       running 1\u002F1   db-1   db-1       healthy  Up 2 hours  postgres:17\n  app    worker   running 1\u002F1   app-1  worker-1   running  Up 2 hours  registry.example.com\u002Fshop\u002Fworker:1.4.2\n",[161,1616,1614],{"__ignoreMap":286},[157,1618,1619,1620,1623,1624,1627,1628,1631],{},"The count under ",[161,1621,1622],{},"STATE"," is the containers that run and pass their health\ncheck, out of those the service should have. A container whose health check\nhas not passed yet makes the service read ",[161,1625,1626],{},"starting",", as in the middle of a\nrollout. A node that does not answer reads ",[161,1629,1630],{},"no answer",", and the reason follows\nthe table:",[281,1633,1636],{"className":1634,"code":1635,"language":488,"meta":286},[486],"  node app-2: accepts SSH, but mmd is not running there (the relay found no daemon socket). Start it on the host with `systemctl start mmd`; `journalctl -u mmd` shows why it stopped\n",[161,1637,1635],{"__ignoreMap":286},[157,1639,1640],{},"The other nodes' rows are shown all the same. Between the two, status lists\nwhat each service name resolves to on each network.",[157,1642,1643,1646,1647,1649],{},[161,1644,1645],{},"service logs web"," merges the lines of every container of ",[161,1648,797],{}," into one\nstream in time order and starts each line with its node and container:",[281,1651,1654],{"className":1652,"code":1653,"language":488,"meta":286},[486],"app-1 web-1 | GET \u002F 200 12ms\napp-2 web-1 | GET \u002Fassets\u002Fapp.css 200 2ms\napp-1 web-1 | GET \u002Fapi\u002Fprojects 200 18ms\n",[161,1655,1653],{"__ignoreMap":286},[157,1657,1658,1659,1662,1663,1666,1667,1670,1671,1674],{},"Without a service name, ",[161,1660,1661],{},"service logs"," reads every service of the environment\nthe same way, and ",[161,1664,1665],{},"--node"," reads only the nodes you name. A node that does not\nanswer is one line in the stream, marked with ",[161,1668,1669],{},"!",", and the command fails only\nwhen no node answered. Lines are ordered by the timestamps the container\nengines record, so two nodes whose clocks differ can swap lines that are\ncloser together than that difference: keep the nodes' clocks synchronised with\nNTP or chrony. ",[173,1672,123],{"href":1673},"..\u002Fruntime-observability"," has\nevery column and option.",[157,1676,1677,1678,1681],{},"Status and logs take no deployment lock, so they answer in the middle of a\nrollout and while a node is down. ",[161,1679,1680],{},"mm service exec"," runs a command in the\nservice's container on the first node of its placement.",[186,1683,1685],{"id":1684},"harden-the-containers","Harden the containers",[157,1687,1688,1689,1691,1692,339],{},"One line in ",[161,1690,202],{}," gives every service of a deployment a read-only\nroot filesystem, no Linux capabilities, no new privileges, the engine's\ndefault seccomp profile, and a writable ",[161,1693,1694],{},"\u002Ftmp",[281,1696,1698],{"className":283,"code":1697,"language":285,"meta":286,"style":286},"hardening: strict\n",[161,1699,1700],{"__ignoreMap":286},[290,1701,1702,1705,1707],{"class":292,"line":293},[290,1703,1704],{"class":303},"hardening",[290,1706,339],{"class":307},[290,1708,1709],{"class":342}," strict\n",[157,1711,1712,1715],{},[161,1713,1714],{},"mm deploy plan"," prints what each service will run with, and the plan hash\ncovers it:",[281,1717,1720],{"className":1718,"code":1719,"language":488,"meta":286},[486],"    security: web: read-only root, no capabilities, no new privileges, default seccomp, tmpfs \u002Ftmp\n",[161,1721,1719],{"__ignoreMap":286},[157,1723,1724],{},"A service that cannot take one of these defaults leaves it out with a reason,\nwhich the plan prints next to it:",[281,1726,1728],{"className":283,"code":1727,"language":285,"meta":286,"style":286},"modules:\n  shop:\n    services:\n      db:\n        stack: app\n        hardening:\n          except:\n            read-only: Postgres writes its socket under \u002Fvar\u002Frun\n",[161,1729,1730,1736,1742,1748,1754,1764,1771,1778],{"__ignoreMap":286},[290,1731,1732,1734],{"class":292,"line":293},[290,1733,500],{"class":303},[290,1735,308],{"class":307},[290,1737,1738,1740],{"class":292,"line":300},[290,1739,823],{"class":303},[290,1741,308],{"class":307},[290,1743,1744,1746],{"class":292,"line":311},[290,1745,842],{"class":303},[290,1747,308],{"class":307},[290,1749,1750,1752],{"class":292,"line":319},[290,1751,884],{"class":303},[290,1753,308],{"class":307},[290,1755,1756,1759,1761],{"class":292,"line":327},[290,1757,1758],{"class":303},"        stack",[290,1760,339],{"class":307},[290,1762,1763],{"class":342}," app\n",[290,1765,1766,1769],{"class":292,"line":361},[290,1767,1768],{"class":303},"        hardening",[290,1770,308],{"class":307},[290,1772,1773,1776],{"class":292,"line":386},[290,1774,1775],{"class":303},"          except",[290,1777,308],{"class":307},[290,1779,1780,1783,1785],{"class":292,"line":411},[290,1781,1782],{"class":303},"            read-only",[290,1784,339],{"class":307},[290,1786,1787],{"class":342}," Postgres writes its socket under \u002Fvar\u002Frun\n",[157,1789,1790,1791,1794,1795,1798,1799,1802,1803,1806,1807,1810,1811,1814],{},"It does not apply to ",[161,1792,1793],{},"mm dev"," on your machine. Images that start as root and\nswitch to their own user, such as the official ",[161,1796,1797],{},"postgres"," and ",[161,1800,1801],{},"nginx"," images,\nneed a few capabilities back with ",[161,1804,1805],{},"cap_add:"," in the Compose file. Each node's\n",[161,1808,1809],{},"\u002Fetc\u002Fmm\u002Fhost.yaml"," can also offer OCI runtimes such as gVisor's ",[161,1812,1813],{},"runsc"," and\nput every service that publishes a port under one of them. Service names,\nshared volumes, and secret files work unchanged under a read-only root.",[186,1816,1818],{"id":1817},"ship-configuration-files","Ship configuration files",[157,1820,1821,1822,1825],{},"A file that is not a secret and not part of the image, such as a proxy\nconfiguration or an init script, goes under the service's ",[161,1823,1824],{},"config-files",".\nEach node that runs the service writes it and mounts it read-only into the\ncontainer:",[281,1827,1829],{"className":283,"code":1828,"language":285,"meta":286,"style":286},"modules:\n  proxy:\n    path: proxy\n    services:\n      proxy:\n        stack: app\n        config-files:\n          \u002Fetc\u002Fnginx\u002Fconf.d\u002Fsite.conf: { template: site.conf.tmpl }\n",[161,1830,1831,1837,1844,1853,1859,1866,1874,1881],{"__ignoreMap":286},[290,1832,1833,1835],{"class":292,"line":293},[290,1834,500],{"class":303},[290,1836,308],{"class":307},[290,1838,1839,1842],{"class":292,"line":300},[290,1840,1841],{"class":303},"  proxy",[290,1843,308],{"class":307},[290,1845,1846,1848,1850],{"class":292,"line":311},[290,1847,831],{"class":303},[290,1849,339],{"class":307},[290,1851,1852],{"class":342}," proxy\n",[290,1854,1855,1857],{"class":292,"line":319},[290,1856,842],{"class":303},[290,1858,308],{"class":307},[290,1860,1861,1864],{"class":292,"line":327},[290,1862,1863],{"class":303},"      proxy",[290,1865,308],{"class":307},[290,1867,1868,1870,1872],{"class":292,"line":361},[290,1869,1758],{"class":303},[290,1871,339],{"class":307},[290,1873,1763],{"class":342},[290,1875,1876,1879],{"class":292,"line":386},[290,1877,1878],{"class":303},"        config-files",[290,1880,308],{"class":307},[290,1882,1883,1886,1888,1890,1893,1895,1898],{"class":292,"line":411},[290,1884,1885],{"class":303},"          \u002Fetc\u002Fnginx\u002Fconf.d\u002Fsite.conf",[290,1887,339],{"class":307},[290,1889,333],{"class":307},[290,1891,1892],{"class":303}," template",[290,1894,339],{"class":307},[290,1896,1897],{"class":342}," site.conf.tmpl",[290,1899,1900],{"class":307}," }\n",[157,1902,1903,1906,1907,1910,1911,1914],{},[161,1904,1905],{},"file:"," ships a file from the module's directory as it is; ",[161,1908,1909],{},"template:"," replaces\neach ",[161,1912,1913],{},"{{ KEY }}"," with the environment's binding of that key. When the file\nchanges, the plan says which services are replaced, and only those roll, one\nnode at a time. A config file is at most 1 MiB and never takes a secret.",[186,1916,1918],{"id":1917},"read-secrets-on-the-nodes","Read secrets on the nodes",[157,1920,1921,1922,1924,1925,1928],{},"Secrets from ",[161,1923,167],{},"'s own store reach the nodes sealed during setup. Secrets\nkept in an external OpenBao or Vault can be read by the nodes themselves, so\ntheir values never pass through the machine that runs ",[161,1926,1927],{},"mm deploy",". Give the\nnodes an AppRole that may only read the environment's secrets, then connect\neach node that runs a service using such a secret, once, as root on the node.\nThe command reads the role ID and the secret ID from stdin, one per line:",[281,1930,1932],{"className":460,"code":1931,"language":462,"meta":286,"style":286},"{\n  bao read -field=role_id auth\u002Fmm-approle\u002Frole\u002Fmm-host-production\u002Frole-id; echo\n  bao write -f -field=secret_id auth\u002Fmm-approle\u002Frole\u002Fmm-host-production\u002Fsecret-id; echo\n} | mmd secrets connect --project PROJECT_ID --environment production \\\n      --provider team-secrets --address https:\u002F\u002F10.90.0.1:8200 \\\n      --ca-file \u002Fetc\u002Fmm\u002Fbao-ca.pem\n",[161,1933,1934,1939,1959,1979,2009,2025],{"__ignoreMap":286},[290,1935,1936],{"class":292,"line":293},[290,1937,1938],{"class":307},"{\n",[290,1940,1941,1944,1947,1950,1953,1956],{"class":292,"line":300},[290,1942,1943],{"class":469},"  bao",[290,1945,1946],{"class":342}," read",[290,1948,1949],{"class":472}," -field=role_id",[290,1951,1952],{"class":342}," auth\u002Fmm-approle\u002Frole\u002Fmm-host-production\u002Frole-id",[290,1954,1955],{"class":307},";",[290,1957,1958],{"class":469}," echo\n",[290,1960,1961,1963,1966,1969,1972,1975,1977],{"class":292,"line":311},[290,1962,1943],{"class":469},[290,1964,1965],{"class":342}," write",[290,1967,1968],{"class":472}," -f",[290,1970,1971],{"class":472}," -field=secret_id",[290,1973,1974],{"class":342}," auth\u002Fmm-approle\u002Frole\u002Fmm-host-production\u002Fsecret-id",[290,1976,1955],{"class":307},[290,1978,1958],{"class":469},[290,1980,1981,1983,1986,1989,1992,1995,1998,2001,2004,2006],{"class":292,"line":319},[290,1982,875],{"class":307},[290,1984,1985],{"class":307}," |",[290,1987,1988],{"class":469}," mmd",[290,1990,1991],{"class":342}," secrets",[290,1993,1994],{"class":342}," connect",[290,1996,1997],{"class":472}," --project",[290,1999,2000],{"class":342}," PROJECT_ID",[290,2002,2003],{"class":472}," --environment",[290,2005,476],{"class":342},[290,2007,2008],{"class":354}," \\\n",[290,2010,2011,2014,2017,2020,2023],{"class":292,"line":327},[290,2012,2013],{"class":472},"      --provider",[290,2015,2016],{"class":342}," team-secrets",[290,2018,2019],{"class":472}," --address",[290,2021,2022],{"class":342}," https:\u002F\u002F10.90.0.1:8200",[290,2024,2008],{"class":354},[290,2026,2027,2030],{"class":292,"line":361},[290,2028,2029],{"class":472},"      --ca-file",[290,2031,2032],{"class":342}," \u002Fetc\u002Fmm\u002Fbao-ca.pem\n",[157,2034,2035],{},"It logs in before it keeps anything and stores the credential encrypted, for\nroot only. A plan then carries the version of every secret each node read,\nand a rotated secret replaces only the services that use it. A node asks the\nstore only for the secrets of its own services, and a node whose services use\nnone needs no connection. When the nodes reach the store at another address\nthan the provider's, such as a node's mesh address, declare it on the target:",[281,2037,2039],{"className":283,"code":2038,"language":285,"meta":286,"style":286},"targets:\n  production:\n    providers:\n      team-secrets: {address: https:\u002F\u002F10.90.0.1:8200}\n",[161,2040,2041,2047,2053,2060],{"__ignoreMap":286},[290,2042,2043,2045],{"class":292,"line":293},[290,2044,304],{"class":303},[290,2046,308],{"class":307},[290,2048,2049,2051],{"class":292,"line":300},[290,2050,314],{"class":303},[290,2052,308],{"class":307},[290,2054,2055,2058],{"class":292,"line":311},[290,2056,2057],{"class":303},"    providers",[290,2059,308],{"class":307},[290,2061,2062,2065,2067,2069,2072,2074,2076],{"class":292,"line":319},[290,2063,2064],{"class":303},"      team-secrets",[290,2066,339],{"class":307},[290,2068,333],{"class":307},[290,2070,2071],{"class":303},"address",[290,2073,339],{"class":307},[290,2075,2022],{"class":342},[290,2077,358],{"class":307},[186,2079,2081],{"id":2080},"deploy-without-git-on-the-nodes","Deploy without git on the nodes",[157,2083,2084,2085,2087],{},"When the nodes may not reach your git server, have ",[161,2086,167],{}," send each commit:",[281,2089,2091],{"className":283,"code":2090,"language":285,"meta":286,"style":286},"targets:\n  production:\n    delivery: push\n    nodes:\n      - {name: app-1, host: 203.0.113.11}\n      - {name: app-2, host: 203.0.113.12}\n      - {name: db-1, host: 203.0.113.13}\n    setup:\n      break-glass:\n        - {name: ops-recovery, key: ssh-ed25519 AAAA… ops@offline}\n",[161,2092,2093,2099,2105,2115,2121,2143,2165,2187,2194,2201],{"__ignoreMap":286},[290,2094,2095,2097],{"class":292,"line":293},[290,2096,304],{"class":303},[290,2098,308],{"class":307},[290,2100,2101,2103],{"class":292,"line":300},[290,2102,314],{"class":303},[290,2104,308],{"class":307},[290,2106,2107,2110,2112],{"class":292,"line":311},[290,2108,2109],{"class":303},"    delivery",[290,2111,339],{"class":307},[290,2113,2114],{"class":342}," push\n",[290,2116,2117,2119],{"class":292,"line":319},[290,2118,322],{"class":303},[290,2120,308],{"class":307},[290,2122,2123,2125,2127,2129,2131,2133,2135,2137,2139,2141],{"class":292,"line":327},[290,2124,330],{"class":307},[290,2126,333],{"class":307},[290,2128,336],{"class":303},[290,2130,339],{"class":307},[290,2132,343],{"class":342},[290,2134,346],{"class":307},[290,2136,349],{"class":303},[290,2138,339],{"class":307},[290,2140,355],{"class":354},[290,2142,358],{"class":307},[290,2144,2145,2147,2149,2151,2153,2155,2157,2159,2161,2163],{"class":292,"line":361},[290,2146,330],{"class":307},[290,2148,333],{"class":307},[290,2150,336],{"class":303},[290,2152,339],{"class":307},[290,2154,372],{"class":342},[290,2156,346],{"class":307},[290,2158,349],{"class":303},[290,2160,339],{"class":307},[290,2162,381],{"class":354},[290,2164,358],{"class":307},[290,2166,2167,2169,2171,2173,2175,2177,2179,2181,2183,2185],{"class":292,"line":386},[290,2168,330],{"class":307},[290,2170,333],{"class":307},[290,2172,336],{"class":303},[290,2174,339],{"class":307},[290,2176,397],{"class":342},[290,2178,346],{"class":307},[290,2180,349],{"class":303},[290,2182,339],{"class":307},[290,2184,406],{"class":354},[290,2186,358],{"class":307},[290,2188,2189,2192],{"class":292,"line":411},[290,2190,2191],{"class":303},"    setup",[290,2193,308],{"class":307},[290,2195,2196,2199],{"class":292,"line":419},[290,2197,2198],{"class":303},"      break-glass",[290,2200,308],{"class":307},[290,2202,2203,2206,2208,2210,2212,2215,2217,2220,2222,2225],{"class":292,"line":426},[290,2204,2205],{"class":307},"        -",[290,2207,333],{"class":307},[290,2209,336],{"class":303},[290,2211,339],{"class":307},[290,2213,2214],{"class":342}," ops-recovery",[290,2216,346],{"class":307},[290,2218,2219],{"class":303}," key",[290,2221,339],{"class":307},[290,2223,2224],{"class":342}," ssh-ed25519 AAAA… ops@offline",[290,2226,358],{"class":307},[157,2228,2229,2230,2233,2234,2236,2237,2239,2240,2243,2244,2246],{},"There is then no ",[161,2231,2232],{},"origin"," and no deploy key in ",[161,2235,451],{},", and the nodes never\nrun git. ",[161,2238,1714],{}," sends every node of the plan a bundle of the files\nthe commit tracks, read from your checkout's git objects, and each node checks\nthem against the commit before it keeps them. A node that already holds the\ncommit is sent nothing, so a rollback costs no upload. Images still come from\ntheir registries, so the nodes need a route to those. With push delivery a\nnode cannot ask the git server which branches hold a commit: ",[161,2241,2242],{},"deploy-from"," is\nchecked against the branches your client states, so give ",[161,2245,962],{}," on such a\ntarget only to CI and to people who may push to those branches.",[186,2248,2250],{"id":2249},"add-or-remove-a-node","Add or remove a node",[157,2252,2253,2254,2256],{},"Add the node to ",[161,2255,163],{},", or delete it, commit, and run setup again:",[281,2258,2260],{"className":460,"code":2259,"language":462,"meta":286,"style":286},"mm -e production target setup plan\nmm -e production target setup apply --plan PLAN_HASH --yes\nmm -e production target setup record\n",[161,2261,2262,2276,2296],{"__ignoreMap":286},[290,2263,2264,2266,2268,2270,2272,2274],{"class":292,"line":293},[290,2265,167],{"class":469},[290,2267,473],{"class":472},[290,2269,476],{"class":342},[290,2271,524],{"class":342},[290,2273,527],{"class":342},[290,2275,557],{"class":342},[290,2277,2278,2280,2282,2284,2286,2288,2290,2292,2294],{"class":292,"line":300},[290,2279,167],{"class":469},[290,2281,473],{"class":472},[290,2283,476],{"class":342},[290,2285,524],{"class":342},[290,2287,527],{"class":342},[290,2289,572],{"class":342},[290,2291,575],{"class":472},[290,2293,578],{"class":342},[290,2295,581],{"class":472},[290,2297,2298,2300,2302,2304,2306,2308],{"class":292,"line":311},[290,2299,167],{"class":469},[290,2301,473],{"class":472},[290,2303,476],{"class":342},[290,2305,524],{"class":342},[290,2307,527],{"class":342},[290,2309,596],{"class":342},[157,2311,2312,2313,2315,2316,208],{},"The plan shows the one mesh entry that changes on every other node. A new node\nruns nothing until a service's ",[161,2314,496],{}," names it and you deploy. Removing a\nnode drops it from the mesh; setup does not contact a host that is no longer\nlisted, and the plan says how to take the mesh off it. Move its services to\nother nodes and deploy before you remove it. See ",[173,2317,2319],{"href":2318},"..\u002Fremote-daemon#add-or-remove-a-node","add or remove a\nnode",[186,2321,2323],{"id":2322},"when-a-node-goes-down","When a node goes down",[157,2325,2326,2327,2329],{},"Nothing is rescheduled. ",[161,2328,167],{}," has no scheduler and no consensus store: the\nservices placed on a node run on that node and nowhere else.",[191,2331,2332,2335,2338,2341,2355],{},[194,2333,2334],{},"The other nodes keep running. Within about ten seconds they stop answering\nservice names with the lost node's containers, so a service that also runs\nelsewhere keeps working by name.",[194,2336,2337],{},"A service that runs only on the lost node is down until the node is back.\nFor a service that has to survive a node, place it on more than one.",[194,2339,2340],{},"When the node boots, the mesh, the firewall rules, and its services come\nback without a deployment.",[194,2342,2343,2344,2346,2347,2350,2351,208],{},"While it is down, ",[161,2345,1714],{}," stops and names the node. To deploy to\nthe others, pass ",[161,2348,2349],{},"--nodes"," with their names; see ",[173,2352,2354],{"href":2353},"..\u002Fremote-daemon#when-a-node-does-not-answer","when a node does not\nanswer",[194,2356,2357],{},"If the lost node is the storage node, reads and writes of shared volumes on\nthe other nodes wait until it returns.",[157,2359,2360,2361,2363,2364,2366],{},"To replace a node for good, remove it from ",[161,2362,163],{},", add its replacement,\nchange the ",[161,2365,496],{}," of its services, run setup, and deploy. Data in its\nlocal volumes is not moved for you.",[186,2368,2370],{"id":2369},"scale-a-service","Scale a service",[157,2372,2373,2375,2376,2378,2379,2382,2383,339],{},[161,2374,934],{}," runs a count of containers spread over the placement's nodes, and\n",[161,2377,938],{}," lets the count follow the load between ",[161,2380,2381],{},"scale.min"," and\n",[161,2384,2385],{},"scale.max",[281,2387,2389],{"className":283,"code":2388,"language":285,"meta":286,"style":286},"modules:\n  shop:\n    services:\n      web:\n        stack: app\n        placement: all\n        autoscale:\n          scale: { min: 1, max: 4 }\n",[161,2390,2391,2397,2403,2409,2415,2423,2433,2440],{"__ignoreMap":286},[290,2392,2393,2395],{"class":292,"line":293},[290,2394,500],{"class":303},[290,2396,308],{"class":307},[290,2398,2399,2401],{"class":292,"line":300},[290,2400,823],{"class":303},[290,2402,308],{"class":307},[290,2404,2405,2407],{"class":292,"line":311},[290,2406,842],{"class":303},[290,2408,308],{"class":307},[290,2410,2411,2413],{"class":292,"line":319},[290,2412,850],{"class":303},[290,2414,308],{"class":307},[290,2416,2417,2419,2421],{"class":292,"line":327},[290,2418,1758],{"class":303},[290,2420,339],{"class":307},[290,2422,1763],{"class":342},[290,2424,2425,2428,2430],{"class":292,"line":361},[290,2426,2427],{"class":303},"        placement",[290,2429,339],{"class":307},[290,2431,2432],{"class":342}," all\n",[290,2434,2435,2438],{"class":292,"line":386},[290,2436,2437],{"class":303},"        autoscale",[290,2439,308],{"class":307},[290,2441,2442,2445,2447,2449,2452,2454,2457,2459,2462,2464,2467],{"class":292,"line":411},[290,2443,2444],{"class":303},"          scale",[290,2446,339],{"class":307},[290,2448,333],{"class":307},[290,2450,2451],{"class":303}," min",[290,2453,339],{"class":307},[290,2455,2456],{"class":354}," 1",[290,2458,346],{"class":307},[290,2460,2461],{"class":303}," max",[290,2463,339],{"class":307},[290,2465,2466],{"class":354}," 4",[290,2468,1900],{"class":307},[157,2470,2471,2472,2476],{},"A container that leaves first drops out of the names and the ingress, drains,\nand then stops, so scaling down drops no request.\n",[173,2473,2475],{"href":2474},"..\u002Fremote-daemon#autoscale-a-service","Autoscale a service"," covers the\nmetrics, the thresholds, and how to watch what the autoscaler decided.",[186,2478,2480],{"id":2479},"restart-a-service-run-a-job-route-traffic-in","Restart a service, run a job, route traffic in",[191,2482,2483,2493,2500],{},[194,2484,2485,2488,2489,208],{},[161,2486,2487],{},"mm service restart"," restarts a service's containers one node at a time; see\n",[173,2490,2492],{"href":2491},"..\u002Fruntime-observability#restart-a-service","restart a service",[194,2494,2495,2496,208],{},"Jobs run on a cluster on their own, with a deployment, or once per\nenvironment; see ",[173,2497,2499],{"href":2498},"..\u002Fjobs-and-schedules#run-a-job-on-a-remote-target","run a job on a remote target",[194,2501,2502,2505,2506,208],{},[161,2503,2504],{},"ingress:"," routes reach the service's containers on every node; see\n",[173,2507,2509],{"href":2508},"..\u002Fremote-daemon#route-traffic-into-the-cluster","route traffic into the cluster",[186,2511,2513],{"id":2512},"what-a-cluster-does-not-do","What a cluster does not do",[191,2515,2516,2519],{},[194,2517,2518],{},"Nothing is rescheduled when a node goes down: its containers come back when\nthe node does, and the other nodes keep answering for services that run on\nthem too.",[194,2520,2521],{},"A named Compose volume is local to its node. Keep a service that stores data\npinned to one node, or use a shared volume for files where speed does not\nmatter.",[157,2523,2524,2528],{},[173,2525,2527],{"href":2526},"..\u002Freference\u002Fknown-issues","Known issues"," lists the remaining limits.",[2530,2531,2532],"style",{},"html pre.shiki code .sP7nD, html code.shiki .sP7nD{--shiki-default:var(--mm-code-comment)}html pre.shiki code .sWZmh, html code.shiki .sWZmh{--shiki-default:var(--mm-code-key)}html pre.shiki code .sIbZ0, html code.shiki .sIbZ0{--shiki-default:var(--mm-code-punctuation)}html pre.shiki code .sv8ni, html code.shiki .sv8ni{--shiki-default:var(--mm-code-string)}html pre.shiki code .s1Wbj, html code.shiki .s1Wbj{--shiki-default:var(--mm-code-constant)}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .svrjQ, html code.shiki .svrjQ{--shiki-default:var(--mm-code-function)}html pre.shiki code .sbRf1, html code.shiki .sbRf1{--shiki-default:var(--mm-code-option)}",{"title":286,"searchDepth":300,"depth":300,"links":2534},[2535,2536,2537,2538,2539,2540,2541,2542,2543,2544,2545,2546,2547,2548,2549,2550,2551,2552],{"id":188,"depth":300,"text":189},{"id":275,"depth":300,"text":276},{"id":504,"depth":300,"text":505},{"id":678,"depth":300,"text":679},{"id":962,"depth":300,"text":963},{"id":1058,"depth":300,"text":1059},{"id":1106,"depth":300,"text":1107},{"id":1350,"depth":300,"text":1351},{"id":1531,"depth":300,"text":1532},{"id":1684,"depth":300,"text":1685},{"id":1817,"depth":300,"text":1818},{"id":1917,"depth":300,"text":1918},{"id":2080,"depth":300,"text":2081},{"id":2249,"depth":300,"text":2250},{"id":2322,"depth":300,"text":2323},{"id":2369,"depth":300,"text":2370},{"id":2479,"depth":300,"text":2480},{"id":2512,"depth":300,"text":2513},"List your servers under one target, set them up, and deploy a Compose project across them.","md",{},{"title":51,"description":2553},"docs\u002Fhow-to\u002Fcluster","AYq8fVl-1V9VaFeG1hs1zCMEgIDkgQ51iKHex7CQqiU",1794776823733]