experiments with ScribbleNET & ControlNet in p5.js

This is a quick setup how to interface with Stable Diffusion / A1111 with simply using p5.js and a locally installed A1111.

let canvas;

const w = 512;
const  h = 768;

 let url = "http://127.0.0.1:7860/controlnet/txt2img"; 
 
 //let url = "http://127.0.0.1:7860/controlnet/txt2img";

let the_drawing_canvas;

let the_drawing_container;

let pos_prompt_input_field;


// ---------------------------------------
// ---------------------------------------
// ---------------------------------------

function hide_drawing_UI(){
  
  the_drawing_container.style.display = 'none';
  
}

function show_drawing_UI(){
  
  the_drawing_container.style.display = 'flex';
  
}


// Function to get the prompt from localStorage
    function getStoredPrompt() {
      let storedPrompt = localStorage.getItem("scribble_pos_prompt");
      if (storedPrompt) {
        pos_prompt_input_field.value = storedPrompt;
      }
    }



// ---------------------------------------
// ---------------------------------------
// ---------------------------------------

function setup() {
  
  pos_prompt_input_field  = document.getElementById("promptInput"); 
  
  getStoredPrompt();
  
  
  the_drawing_container = document.getElementById("canvas_drawing_container");
  
  the_drawing_canvas = createCanvas(w, h);
  
  canvas = createGraphics( w, h );
  
  the_drawing_canvas.parent(the_drawing_container);
 
  
  show_drawing_UI();
  
   canvas.background(255);
  // canvas.endDraw();
}

// ---------------------------------------
// ---------------------------------------
// ---------------------------------------

function draw() {
  background(220);
  
  
  
  if(mouseIsPressed){
    
    canvas.strokeWeight(2);
    canvas.line(mouseX,mouseY,pmouseX,pmouseY)
   // canvas.circle(mouseX,mouseY,2);
    
  }
  
  image(canvas,0,0);
  
  
  
  
}

// ---------------------------------------
// ---------------------------------------
// ---------------------------------------

function sendDIRECT(){
  
  
    hide_drawing_UI();
  
   // After translating the prompt, store it in localStorage
      localStorage.setItem("scribble_pos_prompt", pos_prompt_input_field.value );
  
  let base64String = getBase64StringFromGraphic(canvas);
  
    let imgs  = [];
    imgs[0] = base64String;
      
  
  let all_control_nets = [];
  
  let my_controlnet_unit = {
    
      input_image: base64String,
      
      module: "none",
      model: "control_sd15_scribble",
      weight:1.1,
      resize_mode: 0,
      lowvram:false,
      processor_res:256,
      pixel_perfect:true,
      guidance:0.98,
      guessmode: true
    
 
    
  };
  
  all_control_nets[0] = my_controlnet_unit;
  
  
  let pos_prompt = "((( funky fluffy moss in wooden pool and streamlet,fluffy furry forest floor in background, glossy water drips curly, molten chrome glossy white porcelain metal, complementary pastel color palette unsaturated, surreal painting)))";
  
  pos_prompt = "((( " + pos_prompt_input_field.value + ") )) ";
  
    pos_prompt += ", photorealistic, ultra realistic, maximum detail, foreground focus, epicurious, instagram, 8k, volumetric light, cinematic, octane render, uplight, no blur, depth of field, dof, bokeh, 8k, 4k";

  
  
  // Define the JSON data to be sent
      let data = {
        //init_images:imgs,
        //alwayson_scripts: {},
        prompt: pos_prompt,
        restore_faces: true,
        tiling: false,
        steps:25,
        neg_prompt: "ugly, drawing, label, text, watermark, typo, logo, watermark, text, error, blurry, jpeg artifacts, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, artist name, (worst quality, low quality:1.4), bad anatomy ", 
        width: w,
        height: h,
        batch_size:3,
        sampler_index: "Euler a",
        cfg_scale:5,
        resize_mode:0,
        denoising_strength: 0.77,
        controlnet_units:all_control_nets
      };
  
  
  
  
  // Convert JSON data to a string
      let jsonData = JSON.stringify(data);
  
   //console.log(jsonData);
  
   // Fetch data using the POST method
      fetch(url, {
        method: "POST",
        headers: {
          "Content-Type": "application/json",
        },
        body: jsonData,
      })
        .then(response => response.json())
        .then(handleResponse)
        .catch(error => {
          console.error('Error:', error);
          //hidePreloader();
        });
  
  
}


// ---------------------------------------
// ---------------------------------------
// ---------------------------------------

function handleResponse(response) {
  
    // Clear the existing images container
      let imagesContainer = document.getElementById("images-container");
      imagesContainer.innerHTML = "";
  
  if (response.images && Array.isArray(response.images) && response.images.length > 0) {
        
          /*
          // Assuming the image is in the first element of the "images" array
           let image_raw1 = response.images[0];
        
        // Create an HTML <img> element and set its "src" to the image data
          let img1 = createImg("data:image/png;base64," + image_raw1);
        img1.size(width, height); // Set the size of the image
         img1.position(400, 400); // Position the image on the canvas
        */
    
    
       // If there are more images, handle them here using a loop
        for (let i = 0; i < response.images.length; i++) {
          let image_raw = response.images[i];
          let img = createImg("data:image/png;base64," + image_raw);
          img.size(w, h);
          
          img.parent(imagesContainer);
          
        }
    
    
    
  }
  
  
  
  
}


// ---------------------------------------
// ---------------------------------------
// ---------------------------------------


function getBase64StringFromGraphic(graphic) {
      // Create an offscreen canvas
      let offscreenCanvas = document.createElement('canvas');
      offscreenCanvas.width = graphic.width;
      offscreenCanvas.height = graphic.height;

      // Get the 2D context of the offscreen canvas
      let ctx = offscreenCanvas.getContext('2d');

      // Draw the graphic onto the offscreen canvas
      ctx.drawImage(graphic.elt, 0, 0);

      // Convert the offscreen canvas to base64 string
      let base64String = offscreenCanvas.toDataURL().split(',')[1];

      return base64String;
    }