From 5f71e640176e251f15bbe8852c9ee515984b35ec Mon Sep 17 00:00:00 2001 From: moshferatu Date: Tue, 23 Apr 2024 04:08:04 -0700 Subject: [PATCH] Add attention masking workflow, still a work in progress --- ipadapter/README.md | 6 + ipadapter/ipadapter-attention-masking.json | 1360 ++++++++++++++++++++ 2 files changed, 1366 insertions(+) create mode 100644 ipadapter/ipadapter-attention-masking.json diff --git a/ipadapter/README.md b/ipadapter/README.md index b8fbdf4..c34d8c3 100644 --- a/ipadapter/README.md +++ b/ipadapter/README.md @@ -18,4 +18,10 @@ Credit to Matteo Spinelli (@cubiq) for the original workflows. ![IPAdapter Sketch to Image with Style Transfer](./ipadapter-sketch-to-image-with-style-transfer.png) +## Attention Masking + +I have been unable to get this working for any images / models not used in the [video](https://www.youtube.com/watch?v=4jq6VQHyXjg). + +Still, it's provided here for future reference. + --- \ No newline at end of file diff --git a/ipadapter/ipadapter-attention-masking.json b/ipadapter/ipadapter-attention-masking.json new file mode 100644 index 0000000..a71790e --- /dev/null +++ b/ipadapter/ipadapter-attention-masking.json @@ -0,0 +1,1360 @@ +{ + "last_node_id": 28, + "last_link_id": 39, + "nodes": [ + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 2010, + 250 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 19, + "type": "CLIPTextEncode", + "pos": [ + 1060, + -730 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 39 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 17 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "illustration of a beautiful chinese woman" + ] + }, + { + "id": 22, + "type": "CLIPTextEncode", + "pos": [ + 1060, + -520 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 38 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 19 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 14, + "type": "MaskFromRGBCMYBW+", + "pos": [ + 1280, + -300 + ], + "size": { + "0": 315, + "1": 294 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 10 + } + ], + "outputs": [ + { + "name": "red", + "type": "MASK", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "green", + "type": "MASK", + "links": [ + 14 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "blue", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "cyan", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "magenta", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "yellow", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "black", + "type": "MASK", + "links": null, + "shape": 3 + }, + { + "name": "white", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "MaskFromRGBCMYBW+" + }, + "widgets_values": [ + 0.15, + 0.15, + 0.15, + 0, + false + ] + }, + { + "id": 20, + "type": "CLIPTextEncode", + "pos": [ + 1070, + 70 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 36 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 18 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "illustration of a bald white man" + ] + }, + { + "id": 23, + "type": "CLIPTextEncode", + "pos": [ + 1070, + 280 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 37 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 20 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [ + 560, + 560 + ], + "size": { + "0": 425.27801513671875, + "1": 180.6060791015625 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 28 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, noisy, lowres, bad quality, jpeg, distorted, malformed, msfw" + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 1660, + 240 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 32 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 33 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 34 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 385574056484117, + "randomize", + 20, + 3.5, + "dpmpp_2m_sde", + "karras", + 1 + ] + }, + { + "id": 25, + "type": "ConditioningCombineMultiple+", + "pos": [ + 2080, + -210 + ], + "size": [ + 319.60097045898465, + 106 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "conditioning_1", + "type": "CONDITIONING", + "link": 23 + }, + { + "name": "conditioning_2", + "type": "CONDITIONING", + "link": 24 + }, + { + "name": "conditioning_3", + "type": "CONDITIONING", + "link": 25 + }, + { + "name": "conditioning_4", + "type": "CONDITIONING", + "link": null + }, + { + "name": "conditioning_5", + "type": "CONDITIONING", + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 33 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ConditioningCombineMultiple+" + } + }, + { + "id": 26, + "type": "ConditioningCombineMultiple+", + "pos": [ + 2080, + -40 + ], + "size": { + "0": 319.6009826660156, + "1": 106 + }, + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "conditioning_1", + "type": "CONDITIONING", + "link": 26 + }, + { + "name": "conditioning_2", + "type": "CONDITIONING", + "link": 27 + }, + { + "name": "conditioning_3", + "type": "CONDITIONING", + "link": 28 + }, + { + "name": "conditioning_4", + "type": "CONDITIONING", + "link": null + }, + { + "name": "conditioning_5", + "type": "CONDITIONING", + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 34 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ConditioningCombineMultiple+" + } + }, + { + "id": 24, + "type": "IPAdapterCombineParams", + "pos": [ + 2080, + -390 + ], + "size": { + "0": 317.4000244140625, + "1": 106 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "params_1", + "type": "IPADAPTER_PARAMS", + "link": 21 + }, + { + "name": "params_2", + "type": "IPADAPTER_PARAMS", + "link": 22 + }, + { + "name": "params_3", + "type": "IPADAPTER_PARAMS", + "link": null + }, + { + "name": "params_4", + "type": "IPADAPTER_PARAMS", + "link": null + }, + { + "name": "params_5", + "type": "IPADAPTER_PARAMS", + "link": null + } + ], + "outputs": [ + { + "name": "IPADAPTER_PARAMS", + "type": "IPADAPTER_PARAMS", + "links": [ + 35 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterCombineParams" + } + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 2280, + 150 + ], + "size": { + "0": 573, + "1": 593 + }, + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {}, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [ + 1140, + 510 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 2 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 768, + 512, + 1 + ] + }, + { + "id": 27, + "type": "IPAdapterFromParams", + "pos": [ + 2480, + -170 + ], + "size": { + "0": 315, + "1": 162 + }, + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 29 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": 30 + }, + { + "name": "ipadapter_params", + "type": "IPADAPTER_PARAMS", + "link": 35 + }, + { + "name": "image_negative", + "type": "IMAGE", + "link": null + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 32 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "IPAdapterFromParams" + }, + "widgets_values": [ + "concat", + "V only" + ] + }, + { + "id": 10, + "type": "LoadImage", + "pos": [ + 480, + -430 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "yiyi_attention_mask.png", + "image" + ] + }, + { + "id": 11, + "type": "LoadImage", + "pos": [ + 480, + -60 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 15 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "tt_attention_mask.png", + "image" + ] + }, + { + "id": 12, + "type": "LoadImage", + "pos": [ + 890, + -300 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 10 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "attention_mask.png", + "image" + ] + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + 160, + 430 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 31 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 3, + 5, + 36, + 37, + 38, + 39 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "Art Universe v2.0.safetensors" + ] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [ + 560, + 340 + ], + "size": { + "0": 422.84503173828125, + "1": 164.31304931640625 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 25 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "illustration of a man and a woman, high quality" + ] + }, + { + "id": 17, + "type": "IPAdapterRegionalConditioning", + "pos": [ + 1680, + -420 + ], + "size": { + "0": 317.4000244140625, + "1": 214 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 13 + }, + { + "name": "mask", + "type": "MASK", + "link": 14 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 17 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 19 + } + ], + "outputs": [ + { + "name": "IPADAPTER_PARAMS", + "type": "IPADAPTER_PARAMS", + "links": [ + 21 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "POSITIVE", + "type": "CONDITIONING", + "links": [ + 23 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "NEGATIVE", + "type": "CONDITIONING", + "links": [ + 26 + ], + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "IPAdapterRegionalConditioning" + }, + "widgets_values": [ + 0.9, + 1, + "linear", + 0, + 1 + ] + }, + { + "id": 18, + "type": "IPAdapterRegionalConditioning", + "pos": [ + 1670, + -120 + ], + "size": { + "0": 317.4000244140625, + "1": 214 + }, + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 15 + }, + { + "name": "mask", + "type": "MASK", + "link": 16 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 18 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 20 + } + ], + "outputs": [ + { + "name": "IPADAPTER_PARAMS", + "type": "IPADAPTER_PARAMS", + "links": [ + 22 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "POSITIVE", + "type": "CONDITIONING", + "links": [ + 24 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "NEGATIVE", + "type": "CONDITIONING", + "links": [ + 27 + ], + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "IPAdapterRegionalConditioning" + }, + "widgets_values": [ + 0.9, + 1, + "linear", + 0, + 1 + ] + }, + { + "id": 28, + "type": "IPAdapterUnifiedLoader", + "pos": [ + 2480, + -330 + ], + "size": { + "0": 315, + "1": 78 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 31 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 29 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "ipadapter", + "type": "IPADAPTER", + "links": [ + 30 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "IPAdapterUnifiedLoader" + }, + "widgets_values": [ + "PLUS (high strength)" + ] + } + ], + "links": [ + [ + 2, + 5, + 0, + 3, + 3, + "LATENT" + ], + [ + 3, + 4, + 1, + 6, + 0, + "CLIP" + ], + [ + 5, + 4, + 1, + 7, + 0, + "CLIP" + ], + [ + 7, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 8, + 4, + 2, + 8, + 1, + "VAE" + ], + [ + 9, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 10, + 12, + 0, + 14, + 0, + "IMAGE" + ], + [ + 13, + 10, + 0, + 17, + 0, + "IMAGE" + ], + [ + 14, + 14, + 1, + 17, + 1, + "MASK" + ], + [ + 15, + 11, + 0, + 18, + 0, + "IMAGE" + ], + [ + 16, + 14, + 0, + 18, + 1, + "MASK" + ], + [ + 17, + 19, + 0, + 17, + 2, + "CONDITIONING" + ], + [ + 18, + 20, + 0, + 18, + 2, + "CONDITIONING" + ], + [ + 19, + 22, + 0, + 17, + 3, + "CONDITIONING" + ], + [ + 20, + 23, + 0, + 18, + 3, + "CONDITIONING" + ], + [ + 21, + 17, + 0, + 24, + 0, + "IPADAPTER_PARAMS" + ], + [ + 22, + 18, + 0, + 24, + 1, + "IPADAPTER_PARAMS" + ], + [ + 23, + 17, + 1, + 25, + 0, + "CONDITIONING" + ], + [ + 24, + 18, + 1, + 25, + 1, + "CONDITIONING" + ], + [ + 25, + 6, + 0, + 25, + 2, + "CONDITIONING" + ], + [ + 26, + 17, + 2, + 26, + 0, + "CONDITIONING" + ], + [ + 27, + 18, + 2, + 26, + 1, + "CONDITIONING" + ], + [ + 28, + 7, + 0, + 26, + 2, + "CONDITIONING" + ], + [ + 29, + 28, + 0, + 27, + 0, + "MODEL" + ], + [ + 30, + 28, + 1, + 27, + 1, + "IPADAPTER" + ], + [ + 31, + 4, + 0, + 28, + 0, + "MODEL" + ], + [ + 32, + 27, + 0, + 3, + 0, + "MODEL" + ], + [ + 33, + 25, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 34, + 26, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 35, + 24, + 0, + 27, + 2, + "IPADAPTER_PARAMS" + ], + [ + 36, + 4, + 1, + 20, + 0, + "CLIP" + ], + [ + 37, + 4, + 1, + 23, + 0, + "CLIP" + ], + [ + 38, + 4, + 1, + 22, + 0, + "CLIP" + ], + [ + 39, + 4, + 1, + 19, + 0, + "CLIP" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file