2d image to 3d bas-relief via Sketchup and Ruby

Hello,

So, I saw this video on youtube where the guy takes a 2d image and makes a 3d object out of using a bunch of high end Adobe products and I thought, I bet I could do that with Sketchup and Ruby!

At first I tried just using color detail from the image, and that kinda worked except it wasn’t convincing as a 3d image. So I found this Ai program that will do a reasonable job of creating a depth map image and also is pretty easy to run from the command line: GitHub - isl-org/MiDaS: Code for robust monocular depth estimation described in "Ranftl et. al., Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer, TPAMI 2022"

Here is the original image

And here is the depth map generated by MiDaS

Here is a sample result. I thought it turned out pretty cool!

2d-to-3d-gif

Here is a sketchup model of the result.
2d3d-300-975-025.skp (13.2 MB)

It has some jankiness but I’m pretty happy with it anyway

Here is the script I came up with to turn the 2d image into a 3d bas-relief.

It requires ‘chunky_png’ which is a pure ruby gem because I’m a cheapskate who still uses Sketchup 2017, lol.

Also, I still had to reapply the texture to the finished model by hand as I haven’t quite figured out how to do it with Ruby alone…

So, after the script runs, I had to import the original image, make it a projected texture, switch to parallel camera view and then re-apply the original texture over the finished 3d surface.

require 'logger'
require 'chunky_png'

module JJones

  def self.create_textured_face(model, image_width, image_height, image_path)
    fgroup = model.active_entities.add_group
    entz = fgroup.entities
    
    points = [
      Geom::Point3d.new(0,0,0),
      Geom::Point3d.new(image_width,0,0),
      Geom::Point3d.new(image_width, image_height,0),
      Geom::Point3d.new(0, image_height, 0)
    ]
    face = entz.add_face(points)
    face.reverse! unless face.normal.samedirection?(Z_AXIS)
  
    uvs = [
      Geom::Point3d.new(0,0),
      Geom::Point3d.new(1,0),
      Geom::Point3d.new(1,1),
      Geom::Point3d.new(0,1),
    ]
  
    mapping = [
      points[0], uvs[0],
      points[1], uvs[1],
      points[2], uvs[2],
      points[3], uvs[3],
    ]
  
    material = model.materials.add("Texture Material")
    material.texture = image_path
  
    face.material = Sketchup::Color.new 'Gray'
    face.material = material

    face.position_material(material, mapping, true)
    [face,fgroup]
  end # create_textured_face

  def self.create_grid_textured_face(face, fgroup, image_height, image_width, cell_size, tw, logz=nil)
    model = Sketchup.active_model
    entz = fgroup.entities
    uv_helper = face.get_UVHelper(true, true, tw)

    material = face.material
    texture = material.texture
  
    grid_x = image_height / cell_size 
    grid_y = image_width / cell_size 

    uv_cell_side = cell_size.to_f / image_width.to_f
    logz.info("uv_cell_side: #{uv_cell_side}") if logz
  
    # calculate position of the cell in the texture space
    (0..grid_x).each do |x|
      (0..grid_y).each do |y|
        x1 = x * cell_size
        x2 = (x+1) * cell_size
        y1 = y * cell_size
        y2 = (y+1) * cell_size
        logz.info("#{x1},#{x2},#{y1},#{y2}") if logz
        
        points = [
          Geom::Point3d.new(x1, y1, 0),
          Geom::Point3d.new(x2, y1, 0),
          Geom::Point3d.new(x2, y2, 0),
          Geom::Point3d.new(x1, y2, 0),
        ]
  
        # create new face for the grid cell
        new_face = entz.add_face(points)
  
        # UV coords go from 0,0 in upper left to 1,1 in lower right
        # need to map the grid cell this way
        # x -> 0 -> image_width,  uvx: 0 -> 1
        # uvx cell width = cell_size/image_width
  
        if new_face && new_face.valid?
          uvx1 = x * uv_cell_side
          uvx2 = (x+1) * uv_cell_side
          uvy1 = y * uv_cell_side
          uvy2 = (y+1) * uv_cell_side
          uvs = [
            Geom::Point3d.new(uvx1, uvy1, 0),
            Geom::Point3d.new(uvx2, uvy1, 0),
            Geom::Point3d.new(uvx2, uvy2, 0),
            Geom::Point3d.new(uvx1, uvy2, 0),
          ]
          logz.info("#{uvx1},#{uvx2},#{uvy1},#{uvy2}") if logz
          mapping = [
            points[0], uvs[0],
            points[1], uvs[1],
            points[2], uvs[2],
            points[3], uvs[3],
          ]
          new_face.position_material(material, mapping, true)
        end
      end
    end
  end # create_grid_textured_face
  
  def self.get_pixel_color_from_uv(png_image, u, v, logz=nil)
    # called by get_avg_face_color
    texture_width = png_image.width
    texture_height = png_image.height

    pixel_x = (u * texture_width).to_i
    pixel_y = (v * texture_height).to_i
    if (pixel_x >= texture_width) || (pixel_y >= texture_height) || (pixel_x < 0) || (pixel_y < 0)
      logz.info("pixel(s) #{pixel_x}, #{pixel_y} out of bounds for image?") if logz
      return nil
    end

    # sample the color at these pixel coords
    pixel_color = png_image[pixel_x, pixel_y]
    r = ChunkyPNG::Color.r(pixel_color)
    g = ChunkyPNG::Color.g(pixel_color)
    b = ChunkyPNG::Color.b(pixel_color)
    a = ChunkyPNG::Color.a(pixel_color)
    return [r,g,b,a]
  end # get_pixel_color_from_uv
  
  def self.get_uvz(front, faceEnt, tw)
    # called by get_avg_face_color
    verts = faceEnt.outer_loop.vertices
    points = verts.collect { |v| v.position }
    # uv helper allows you to determine the location (UV coordinates) of a texture on a face
    uvh = faceEnt.get_UVHelper(true, true, tw)
    if front
      uvs = points.collect {|p| uvh.get_front_UVQ(p)}
    else
      uvs = points.collect {|p| uvh.get_back_UVQ(p)}
    end

    for i in (0..uvs.length-1)
      uvs[i] = uvs[i].to_a # convert point3d to array
      uv_w = uvs[i].z
      uvs[i].x = uvs[i].x / uv_w
      uvs[i].y = uvs[i].y / uv_w
      uvs[i].y = uvs[i].y / uv_w
    end
    return uvs
  end # get_uvz
  
  def self.get_average_color(colors)
    # called by get_avg_face_color
    num_pixels = colors.length
    if num_pixels == 0
      print("get_average_color: returning nil")
      return nil
    end
    total_color = { r: 0, g: 0, b: 0, a: 0 }
   
     colors.each do |color|
       total_color[:r] += color[0]
       total_color[:g] += color[1]
       total_color[:b] += color[2]
       total_color[:a] += color[3]
    end
   
    avg_color = {
      r: total_color[:r] / num_pixels,
      g: total_color[:g] / num_pixels,
      b: total_color[:b] / num_pixels,
      a: total_color[:a] / num_pixels
    }
   
    result = [avg_color[:r],avg_color[:g],avg_color[:b],avg_color[:a]]
    return result
  end # get_average_color
 
  def self.get_avg_face_color(image, face, tw, logz=nil)
    # called by calculate_average_color_around_vertex
    # compute the average color based on corner points of face
    # doing front only unless we need back, then changes are required
    colors = []
    uvs = get_uvz(true, face, tw)
    uvs.each do |p|
      u,v = p
      pixel_color = get_pixel_color_from_uv(image, u, v, logz)
      if ! pixel_color.nil?
        colors << pixel_color
      end
    end
    avg_color = get_average_color(colors)
    return avg_color
  end # get_avg_face_color

  def self.calculate_average_color_around_vertex(vertex, image, tw, logz=nil)
    # called by calculate_lightness_scores
    if vertex.nil?
      print("vertex was nil?")
      return nil
    end
    r_total = 0
    g_total = 0
    b_total = 0
    a_total = 0

    # determine the average color of the faces around this vertex
    facez = vertex.faces
    num_faces = facez.length
    facez.each_with_index do |face, i|
      texture = face.material.texture
      avg_color = get_avg_face_color(image, face, tw)
      if avg_color.nil?
        logz.info("avg_color nil for face: #{i}") if logz
        next
      end
      r,g,b,a = avg_color.to_a
      r_total += r
      g_total += g
      b_total += b
      a_total += a
    end
    r_avg = r_total.to_f / num_faces.to_f
    g_avg = g_total.to_f / num_faces.to_f
    b_avg = b_total.to_f / num_faces.to_f
    a_avg = a_total.to_f / num_faces.to_f
    vertex_color_avg = Sketchup::Color.new(r_avg.to_i, g_avg.to_i, b_avg.to_i)
    return vertex_color_avg
  end # calculate_average_color_around_vertex
  
  def self.swap_image_y_direction(image_png)
    # called by load_texture_png
    # create a new blank image
    # copy the pixels over to the new image
    # but swap the y direction
    width = image_png.width
    height = image_png .height
    new_image = ChunkyPNG::Image.new(width, height)
    new_y = 0
    last_row = height-1
    last_row.step(0, -1) do |y|
      image_png.row(y).each_with_index do |pixel,x|
        pixel_color = image_png[x, y]
        new_image[x,new_y] = pixel_color
      end
      new_y += 1
    end
    new_image
  end # swap_image_y_direction

  def self.load_texture_png(texture_file_name)
    # called by main
    # load an image using ChunkyPNG
    image = ChunkyPNG::Image.from_file(texture_file_name)
    new_image = self.swap_image_y_direction(image)
    return new_image
  end # load_texture_png

  def self.get_lightness_from_color(color)
    # called by calculate_lightness_scores
    r,g,b,a = color.to_a
    lightness = (0.2126 * r) + (0.7152 * g) + (0.0722 * b)
    return lightness
  end # get_lightness_from_color
  
  def self.get_interpolated_value(value, max, scale, logz=nil)
    # called by calculate_vertz_height_by_light
    logz.info("get_interpolated_value: value: #{value}, max: #{max}, scale: #{scale}") if logz
    if value.nil? || max.nil? || scale.nil?
      print("get_interpolated_value: something was nil")
      return nil
    end
    n = value / max.to_f
    n = n * scale
    return n
  end # get_interpolated_value
  
  def self.get_average_from_array(arr)
    # called by calculate_vertz_height_by_light
    total = 0
    len = arr.length
    arr.each do |v|
      total += v
    end
    return total / len.to_f
  end # get_average_from_array
  
  def self.smooth_edges(shape, smooth=true)
    # called by main
    edges = shape.definition.entities.grep(Sketchup::Edge)
    edges.each do |edge|
      edge.soft = smooth
      edge.smooth = smooth
    end
  end # smooth_edges

  def self.update_vert_color_map(vertex, color, map)
    # called by calculate_lightness_scores
    if ! map.has_key?(vertex)
      map[vertex] = []
    end
    colors = map[vertex]
    if ! colors.include?(color)
      colors << color
    end
  end # update_vert_color_map

  def self.update_map_counter(obj, map)
    # s = color.to_s
    if ! map.has_key?(obj)
      map[obj] = 0
    end
    map[obj] += 1
  end

  def self.calculate_lightness_scores(selection, image, tw, name, logz=nil)
    # for all the vertices in the selected object
    #     calculate average color around the vertex based on the png image pixel values
    #     record the 'lightness' score
    entz = selection.entities
    faces = entz.grep(Sketchup::Face)
    vert_light_map = {}
    light_scores_count_map = {}
    faces.each_with_index do |face, i|
      if (i % 100) == 0
        logz.info("i: #{i}") if logz
      end
      vertz = face.outer_loop.vertices
      vertz.each do |vertex|
        avg_color = calculate_average_color_around_vertex(vertex, image, tw, logz)
        if avg_color.nil?
          logz.info("avg_color was nil?") if logz
          next
        end
        lightness_score = get_lightness_from_color(avg_color)
        update_vert_color_map(vertex, lightness_score, vert_light_map)
        update_map_counter(lightness_score, light_scores_count_map)
      end
    end
    return [vert_light_map, light_scores_count_map]
  end # calculate_lightness_scores
  
  def self.calculate_vertz_height_by_light(vert_light_map, depth_light_map, 
      height_scale, carrier_weight=0.975, detail_weight=0.025, logz=nil)
    darkest = 0.0
    lightest = 256.0
    
    # return array of transformations
    # darkests: 0.0, lightest: 215.0642
    # from darkest to lightest: interpolate via divide each number by max and multiply by height_scale
    # for each vertex, build an xform to raise it by the interpolated lightness score
    print("calculate_vertz_height_by_light: scale: #{height_scale}, logz: #{logz}")
    vertz = vert_light_map.keys
    xforms = []
    vertz.each do |vertex|
      color_light = vert_light_map[vertex]
      depth_light = depth_light_map[vertex]
      color_value = get_average_from_array(color_light)
      depth_value = get_average_from_array(depth_light)
      color_height = get_interpolated_value(color_value, lightest, height_scale, logz)
      depth_height = get_interpolated_value(depth_value, lightest, height_scale, logz)
      if color_height.nil? || depth_height.nil?
        logz.info("nil height for: #{vertex}, #{color_height}, #{depth_height}") if logz
      end
      
      height = (carrier_weight * depth_height) + (detail_weight * color_height)
      logz.info("height: #{height}, color_height: #{color_height}, depth_height: #{depth_height}") if logz
      
      position = vertex.position
      position.z = height
      vector = vertex.position.vector_to(position)
      xforms << vector
    end
    return xforms
  end # calculate_vertz_height_by_light
  
  def self.create_3d_depth_map(selection, image, depth_map, tw, 
      height_scale, carrier_weight, detail_weight, logz=nil)
    logz.info("BEGIN create_3d_depth_map") if logz
    
    entz = selection.entities

    image_vert_light_map, image_light_counts = calculate_lightness_scores(selection, image, tw, "image", logz)
    depth_vert_light_map, depth_light_counts = calculate_lightness_scores(selection, depth_map, tw, "depth", logz)
    
    image_vertex_xforms = calculate_vertz_height_by_light(image_vert_light_map, depth_vert_light_map, height_scale, carrier_weight, detail_weight, logz)
    vertz = image_vert_light_map.keys
    entz.transform_by_vectors(vertz, image_vertex_xforms)
  end # create_3d_depth_map
  
  def self.main()
    print("main")
    
    model = Sketchup.active_model
    logger = Logger.new("/Users/jjones/src/sketchup/ruby_scripts/log.txt")

    texture_file_name = "/Users/jjones/Documents/texture.png" 
    depth_map_file = "/Users/jjones/Documents/midas-depth.png" 
    
    tw=Sketchup.create_texture_writer
    cell_size = 4
    image = load_texture_png(texture_file_name)
    depth_map = load_texture_png(depth_map_file)
    image_height = image.height
    image_width = image.width
    
    model.active_view.refresh

    model.start_operation "create gridded texture"
      print("create gridded texture")
      face, fgroup = create_textured_face(model, image_width, image_height, texture_file_name)
      create_grid_textured_face(face, fgroup, image_height, image_width, cell_size, tw, logz=logger)
      # select the face group
      model.selection.add(fgroup)
    model.commit_operation
    
    model.start_operation "create 3d depth map"
      print("create 3d depth map")
      model.active_view.refresh
      selection = model.selection
      print("selection: #{selection}, #{selection.class}")
      return unless selection.any?
      group = selection[0]
      height_scale = 300
      carrier_weight = 0.975
      detail_weight = 0.025
      create_3d_depth_map(group, image, depth_map, tw, height_scale, carrier_weight, detail_weight, logz=logger)
    model.commit_operation

    model.start_operation "smooth edges"
      model.active_view.refresh
      selection = model.selection
      print("selection: #{selection}, #{selection.class}")
      return unless selection.any?
      group = selection[0]
      smooth_edges(group)
    model.commit_operation

    model.start_operation "reapply texture"
      # todo
    model.commit_operation

    logger.close
  end # main
end # module

JJones.main

print("All done!")
9 Likes