Upload diff-diff module

__init__.py

@@ -0,0 +1,9 @@
+from diffusers.modular_pipelines.node_utils import ModularNode
+
+
+
+MODULE_MAP = {
+    "DiffDiff": ModularNode.load_mellon_config("custom/diff-diff/diffdiff_mellon_config.json")["module"],
+    "DiffDiffDenoise": ModularNode.load_mellon_config("custom/diff-diff/diffdiff_denoise_mellon_config.json")["module"],
+}
+

diff-diff.py

@@ -0,0 +1,42 @@
+import logging
+from mellon.NodeBase import NodeBase
+
+# Configure logger
+logger = logging.getLogger("mellon")
+logger.setLevel(logging.DEBUG)
+
+
+
+
+# Initialize components manager
+# components = ComponentsManager()
+from custom import components
+
+
+class DiffDiff(NodeBase):
+    def __init__(self, node_id=None):
+        super().__init__(node_id)
+        from diffusers.modular_pipelines.node_utils import ModularNode
+        self._diffdiff_block = ModularNode.from_pretrained("YiYiXu/modular-diffdiff", trust_remote_code=True)
+        self._diffdiff_block.setup(components=components)
+
+    def execute(self, **kwargs):
+        return self._diffdiff_block.execute(**kwargs)
+
+
+class DiffDiffDenoise(NodeBase):
+    def __init__(self, node_id=None):
+        super().__init__(node_id)
+        from diffusers.modular_pipelines.node_utils import ModularNode
+        from diffusers.modular_pipelines import ModularPipelineMixin
+        diffdiff = ModularPipelineMixin.from_pretrained("YiYiXu/modular-diffdiff", trust_remote_code=True)
+        diffdiff.blocks.pop("text_encoder")
+        diffdiff.blocks.pop("decode")
+        diffdiff.blocks.pop("ip_adapter")
+        self._diffdiff_block = ModularNode(diffdiff)
+        self._diffdiff_block.setup(components=components)
+
+    def execute(self, **kwargs):
+        return self._diffdiff_block.execute(**kwargs)
+
+

diffdiff_denoise_mellon_config.json

@@ -0,0 +1,93 @@
+{
+  "module": {
+    "label": "DiffDiffDenoiseBlocks",
+    "category": "Modular Diffusers",
+    "params": {
+      "image": {
+        "label": "Image",
+        "type": "image",
+        "display": "input"
+      },
+      "height": {
+        "label": "Height",
+        "type": "int",
+        "display": "text",
+        "default": 1024,
+        "min": 8,
+        "max": 8192,
+        "step": 8,
+        "group": "dimensions"
+      },
+      "width": {
+        "label": "Width",
+        "type": "int",
+        "display": "text",
+        "default": 1024,
+        "min": 8,
+        "max": 8192,
+        "step": 8,
+        "group": "dimensions"
+      },
+      "num_inference_steps": {
+        "label": "Steps",
+        "type": "int",
+        "default": 25,
+        "min": 1,
+        "max": 1000
+      },
+      "diffdiff_map": {
+        "type": "image",
+        "label": "diffdiff_map",
+        "display": "input"
+      },
+      "prompt_embeddings": {
+        "label": "prompt_embeddings",
+        "type": "prompt_embeddings",
+        "display": "input"
+      },
+      "ip_adapter_embeds": {
+        "label": "ip_adapter_embeds",
+        "type": "ip_adapter_embeds",
+        "display": "input"
+      },
+      "vae": {
+        "label": "vae",
+        "type": "diffusers_auto_model",
+        "display": "input"
+      },
+      "scheduler": {
+        "label": "scheduler",
+        "type": "scheduler",
+        "display": "input"
+      },
+      "guider": {
+        "label": "guider",
+        "type": "guider",
+        "display": "input"
+      },
+      "unet": {
+        "label": "unet",
+        "type": "diffusers_auto_model",
+        "display": "input"
+      },
+      "controlnet": {
+        "label": "controlnet",
+        "type": "diffusers_auto_model",
+        "display": "input"
+      },
+      "latents": {
+        "label": "latents",
+        "type": "latents",
+        "display": "output"
+      }
+    }
+  },
+  "name_mapping": {
+    "prompt_embeds": "prompt_embeddings",
+    "negative_prompt_embeds": "prompt_embeddings",
+    "pooled_prompt_embeds": "prompt_embeddings",
+    "negative_pooled_prompt_embeds": "prompt_embeddings",
+    "ip_adapter_embeds": "ip_adapter_embeds",
+    "negative_ip_adapter_embeds": "ip_adapter_embeds"
+  }
+}

diffdiff_mellon_config.json

@@ -0,0 +1,110 @@
+{
+  "module": {
+    "label": "DiffDiffBlocks",
+    "category": "Modular Diffusers",
+    "params": {
+      "prompt": {
+        "label": "Prompt",
+        "type": "string",
+        "default": "a bear sitting in a chair drinking a milkshake",
+        "display": "textarea"
+      },
+      "negative_prompt": {
+        "label": "Negative Prompt",
+        "type": "string",
+        "default": "deformed, ugly, wrong proportion, low res, bad anatomy, worst quality, low quality",
+        "display": "textarea"
+      },
+      "image": {
+        "label": "Image",
+        "type": "image",
+        "display": "input"
+      },
+      "height": {
+        "label": "Height",
+        "type": "int",
+        "display": "text",
+        "default": 1024,
+        "min": 8,
+        "max": 8192,
+        "step": 8,
+        "group": "dimensions"
+      },
+      "width": {
+        "label": "Width",
+        "type": "int",
+        "display": "text",
+        "default": 1024,
+        "min": 8,
+        "max": 8192,
+        "step": 8,
+        "group": "dimensions"
+      },
+      "num_inference_steps": {
+        "label": "Steps",
+        "type": "int",
+        "default": 25,
+        "min": 1,
+        "max": 1000
+      },
+      "diffdiff_map": {
+        "type": "image",
+        "label": "diffdiff_map",
+        "display": "input"
+      },
+      "ip_adapter_embeds": {
+        "label": "ip_adapter_embeds",
+        "type": "ip_adapter_embeds",
+        "display": "input"
+      },
+      "text_encoders": {
+        "label": "text_encoders",
+        "type": "text_encoders",
+        "display": "input"
+      },
+      "guider": {
+        "label": "guider",
+        "type": "guider",
+        "display": "input"
+      },
+      "image_encoder": {
+        "label": "image_encoder",
+        "type": "diffusers_auto_model",
+        "display": "input"
+      },
+      "unet": {
+        "label": "unet",
+        "type": "diffusers_auto_model",
+        "display": "input"
+      },
+      "vae": {
+        "label": "vae",
+        "type": "diffusers_auto_model",
+        "display": "input"
+      },
+      "scheduler": {
+        "label": "scheduler",
+        "type": "scheduler",
+        "display": "input"
+      },
+      "controlnet": {
+        "label": "controlnet",
+        "type": "diffusers_auto_model",
+        "display": "input"
+      },
+      "images": {
+        "label": "Images",
+        "type": "image",
+        "display": "output"
+      }
+    }
+  },
+  "name_mapping": {
+    "ip_adapter_embeds": "ip_adapter_embeds",
+    "negative_ip_adapter_embeds": "ip_adapter_embeds",
+    "text_encoder": "text_encoders",
+    "text_encoder_2": "text_encoders",
+    "tokenizer": "text_encoders",
+    "tokenizer_2": "text_encoders"
+  }
+}

save_mellon_config.py

@@ -0,0 +1,24 @@
+# YiYi Notes: use this to generate mellon config
+# TODO: 
+# 1. the whole process to create a custom node should be automated with something like `node.push_to_hub()`
+# 2. should be able to edit mellon_config.json
+
+
+from diffusers.modular_pipelines.node_utils import ModularNode
+from diffusers.modular_pipelines import ModularPipelineMixin
+diffdiff = ModularPipelineMixin.from_pretrained("YiYiXu/modular-diffdiff", trust_remote_code=True)
+
+# this is the complete pipeline
+node = ModularNode(diffdiff)
+node.save_mellon_config("diffdiff_mellon_config.json")
+
+
+# this is the denoise pipeline, to be used in the modular setting, 
+# remove the text_encoder, decode, and ip_adapter blocks since we already have these nodes in official modular diffusers mellon
+diffdiff.blocks.pop("text_encoder")
+diffdiff.blocks.pop("decode")
+diffdiff.blocks.pop("ip_adapter")
+
+node = ModularNode(diffdiff, label = "DiffDiffDenoiseBlocks")
+node.save_mellon_config("diffdiff_denoise_mellon_config.json")
+