Hello,
So, I saw this video on youtube where the guy takes a 2d image and makes a 3d object out of using a bunch of high end Adobe products and I thought, I bet I could do that with Sketchup and Ruby!
At first I tried just using color detail from the image, and that kinda worked except it wasn’t convincing as a 3d image. So I found this Ai program that will do a reasonable job of creating a depth map image and also is pretty easy to run from the command line: GitHub - isl-org/MiDaS: Code for robust monocular depth estimation described in "Ranftl et. al., Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer, TPAMI 2022"
Here is the original image
And here is the depth map generated by MiDaS
Here is a sample result. I thought it turned out pretty cool!
Here is a sketchup model of the result.
2d3d-300-975-025.skp (13.2 MB)
It has some jankiness but I’m pretty happy with it anyway
Here is the script I came up with to turn the 2d image into a 3d bas-relief.
It requires ‘chunky_png’ which is a pure ruby gem because I’m a cheapskate who still uses Sketchup 2017, lol.
Also, I still had to reapply the texture to the finished model by hand as I haven’t quite figured out how to do it with Ruby alone…
So, after the script runs, I had to import the original image, make it a projected texture, switch to parallel camera view and then re-apply the original texture over the finished 3d surface.
require 'logger'
require 'chunky_png'
module JJones
def self.create_textured_face(model, image_width, image_height, image_path)
fgroup = model.active_entities.add_group
entz = fgroup.entities
points = [
Geom::Point3d.new(0,0,0),
Geom::Point3d.new(image_width,0,0),
Geom::Point3d.new(image_width, image_height,0),
Geom::Point3d.new(0, image_height, 0)
]
face = entz.add_face(points)
face.reverse! unless face.normal.samedirection?(Z_AXIS)
uvs = [
Geom::Point3d.new(0,0),
Geom::Point3d.new(1,0),
Geom::Point3d.new(1,1),
Geom::Point3d.new(0,1),
]
mapping = [
points[0], uvs[0],
points[1], uvs[1],
points[2], uvs[2],
points[3], uvs[3],
]
material = model.materials.add("Texture Material")
material.texture = image_path
face.material = Sketchup::Color.new 'Gray'
face.material = material
face.position_material(material, mapping, true)
[face,fgroup]
end # create_textured_face
def self.create_grid_textured_face(face, fgroup, image_height, image_width, cell_size, tw, logz=nil)
model = Sketchup.active_model
entz = fgroup.entities
uv_helper = face.get_UVHelper(true, true, tw)
material = face.material
texture = material.texture
grid_x = image_height / cell_size
grid_y = image_width / cell_size
uv_cell_side = cell_size.to_f / image_width.to_f
logz.info("uv_cell_side: #{uv_cell_side}") if logz
# calculate position of the cell in the texture space
(0..grid_x).each do |x|
(0..grid_y).each do |y|
x1 = x * cell_size
x2 = (x+1) * cell_size
y1 = y * cell_size
y2 = (y+1) * cell_size
logz.info("#{x1},#{x2},#{y1},#{y2}") if logz
points = [
Geom::Point3d.new(x1, y1, 0),
Geom::Point3d.new(x2, y1, 0),
Geom::Point3d.new(x2, y2, 0),
Geom::Point3d.new(x1, y2, 0),
]
# create new face for the grid cell
new_face = entz.add_face(points)
# UV coords go from 0,0 in upper left to 1,1 in lower right
# need to map the grid cell this way
# x -> 0 -> image_width, uvx: 0 -> 1
# uvx cell width = cell_size/image_width
if new_face && new_face.valid?
uvx1 = x * uv_cell_side
uvx2 = (x+1) * uv_cell_side
uvy1 = y * uv_cell_side
uvy2 = (y+1) * uv_cell_side
uvs = [
Geom::Point3d.new(uvx1, uvy1, 0),
Geom::Point3d.new(uvx2, uvy1, 0),
Geom::Point3d.new(uvx2, uvy2, 0),
Geom::Point3d.new(uvx1, uvy2, 0),
]
logz.info("#{uvx1},#{uvx2},#{uvy1},#{uvy2}") if logz
mapping = [
points[0], uvs[0],
points[1], uvs[1],
points[2], uvs[2],
points[3], uvs[3],
]
new_face.position_material(material, mapping, true)
end
end
end
end # create_grid_textured_face
def self.get_pixel_color_from_uv(png_image, u, v, logz=nil)
# called by get_avg_face_color
texture_width = png_image.width
texture_height = png_image.height
pixel_x = (u * texture_width).to_i
pixel_y = (v * texture_height).to_i
if (pixel_x >= texture_width) || (pixel_y >= texture_height) || (pixel_x < 0) || (pixel_y < 0)
logz.info("pixel(s) #{pixel_x}, #{pixel_y} out of bounds for image?") if logz
return nil
end
# sample the color at these pixel coords
pixel_color = png_image[pixel_x, pixel_y]
r = ChunkyPNG::Color.r(pixel_color)
g = ChunkyPNG::Color.g(pixel_color)
b = ChunkyPNG::Color.b(pixel_color)
a = ChunkyPNG::Color.a(pixel_color)
return [r,g,b,a]
end # get_pixel_color_from_uv
def self.get_uvz(front, faceEnt, tw)
# called by get_avg_face_color
verts = faceEnt.outer_loop.vertices
points = verts.collect { |v| v.position }
# uv helper allows you to determine the location (UV coordinates) of a texture on a face
uvh = faceEnt.get_UVHelper(true, true, tw)
if front
uvs = points.collect {|p| uvh.get_front_UVQ(p)}
else
uvs = points.collect {|p| uvh.get_back_UVQ(p)}
end
for i in (0..uvs.length-1)
uvs[i] = uvs[i].to_a # convert point3d to array
uv_w = uvs[i].z
uvs[i].x = uvs[i].x / uv_w
uvs[i].y = uvs[i].y / uv_w
uvs[i].y = uvs[i].y / uv_w
end
return uvs
end # get_uvz
def self.get_average_color(colors)
# called by get_avg_face_color
num_pixels = colors.length
if num_pixels == 0
print("get_average_color: returning nil")
return nil
end
total_color = { r: 0, g: 0, b: 0, a: 0 }
colors.each do |color|
total_color[:r] += color[0]
total_color[:g] += color[1]
total_color[:b] += color[2]
total_color[:a] += color[3]
end
avg_color = {
r: total_color[:r] / num_pixels,
g: total_color[:g] / num_pixels,
b: total_color[:b] / num_pixels,
a: total_color[:a] / num_pixels
}
result = [avg_color[:r],avg_color[:g],avg_color[:b],avg_color[:a]]
return result
end # get_average_color
def self.get_avg_face_color(image, face, tw, logz=nil)
# called by calculate_average_color_around_vertex
# compute the average color based on corner points of face
# doing front only unless we need back, then changes are required
colors = []
uvs = get_uvz(true, face, tw)
uvs.each do |p|
u,v = p
pixel_color = get_pixel_color_from_uv(image, u, v, logz)
if ! pixel_color.nil?
colors << pixel_color
end
end
avg_color = get_average_color(colors)
return avg_color
end # get_avg_face_color
def self.calculate_average_color_around_vertex(vertex, image, tw, logz=nil)
# called by calculate_lightness_scores
if vertex.nil?
print("vertex was nil?")
return nil
end
r_total = 0
g_total = 0
b_total = 0
a_total = 0
# determine the average color of the faces around this vertex
facez = vertex.faces
num_faces = facez.length
facez.each_with_index do |face, i|
texture = face.material.texture
avg_color = get_avg_face_color(image, face, tw)
if avg_color.nil?
logz.info("avg_color nil for face: #{i}") if logz
next
end
r,g,b,a = avg_color.to_a
r_total += r
g_total += g
b_total += b
a_total += a
end
r_avg = r_total.to_f / num_faces.to_f
g_avg = g_total.to_f / num_faces.to_f
b_avg = b_total.to_f / num_faces.to_f
a_avg = a_total.to_f / num_faces.to_f
vertex_color_avg = Sketchup::Color.new(r_avg.to_i, g_avg.to_i, b_avg.to_i)
return vertex_color_avg
end # calculate_average_color_around_vertex
def self.swap_image_y_direction(image_png)
# called by load_texture_png
# create a new blank image
# copy the pixels over to the new image
# but swap the y direction
width = image_png.width
height = image_png .height
new_image = ChunkyPNG::Image.new(width, height)
new_y = 0
last_row = height-1
last_row.step(0, -1) do |y|
image_png.row(y).each_with_index do |pixel,x|
pixel_color = image_png[x, y]
new_image[x,new_y] = pixel_color
end
new_y += 1
end
new_image
end # swap_image_y_direction
def self.load_texture_png(texture_file_name)
# called by main
# load an image using ChunkyPNG
image = ChunkyPNG::Image.from_file(texture_file_name)
new_image = self.swap_image_y_direction(image)
return new_image
end # load_texture_png
def self.get_lightness_from_color(color)
# called by calculate_lightness_scores
r,g,b,a = color.to_a
lightness = (0.2126 * r) + (0.7152 * g) + (0.0722 * b)
return lightness
end # get_lightness_from_color
def self.get_interpolated_value(value, max, scale, logz=nil)
# called by calculate_vertz_height_by_light
logz.info("get_interpolated_value: value: #{value}, max: #{max}, scale: #{scale}") if logz
if value.nil? || max.nil? || scale.nil?
print("get_interpolated_value: something was nil")
return nil
end
n = value / max.to_f
n = n * scale
return n
end # get_interpolated_value
def self.get_average_from_array(arr)
# called by calculate_vertz_height_by_light
total = 0
len = arr.length
arr.each do |v|
total += v
end
return total / len.to_f
end # get_average_from_array
def self.smooth_edges(shape, smooth=true)
# called by main
edges = shape.definition.entities.grep(Sketchup::Edge)
edges.each do |edge|
edge.soft = smooth
edge.smooth = smooth
end
end # smooth_edges
def self.update_vert_color_map(vertex, color, map)
# called by calculate_lightness_scores
if ! map.has_key?(vertex)
map[vertex] = []
end
colors = map[vertex]
if ! colors.include?(color)
colors << color
end
end # update_vert_color_map
def self.update_map_counter(obj, map)
# s = color.to_s
if ! map.has_key?(obj)
map[obj] = 0
end
map[obj] += 1
end
def self.calculate_lightness_scores(selection, image, tw, name, logz=nil)
# for all the vertices in the selected object
# calculate average color around the vertex based on the png image pixel values
# record the 'lightness' score
entz = selection.entities
faces = entz.grep(Sketchup::Face)
vert_light_map = {}
light_scores_count_map = {}
faces.each_with_index do |face, i|
if (i % 100) == 0
logz.info("i: #{i}") if logz
end
vertz = face.outer_loop.vertices
vertz.each do |vertex|
avg_color = calculate_average_color_around_vertex(vertex, image, tw, logz)
if avg_color.nil?
logz.info("avg_color was nil?") if logz
next
end
lightness_score = get_lightness_from_color(avg_color)
update_vert_color_map(vertex, lightness_score, vert_light_map)
update_map_counter(lightness_score, light_scores_count_map)
end
end
return [vert_light_map, light_scores_count_map]
end # calculate_lightness_scores
def self.calculate_vertz_height_by_light(vert_light_map, depth_light_map,
height_scale, carrier_weight=0.975, detail_weight=0.025, logz=nil)
darkest = 0.0
lightest = 256.0
# return array of transformations
# darkests: 0.0, lightest: 215.0642
# from darkest to lightest: interpolate via divide each number by max and multiply by height_scale
# for each vertex, build an xform to raise it by the interpolated lightness score
print("calculate_vertz_height_by_light: scale: #{height_scale}, logz: #{logz}")
vertz = vert_light_map.keys
xforms = []
vertz.each do |vertex|
color_light = vert_light_map[vertex]
depth_light = depth_light_map[vertex]
color_value = get_average_from_array(color_light)
depth_value = get_average_from_array(depth_light)
color_height = get_interpolated_value(color_value, lightest, height_scale, logz)
depth_height = get_interpolated_value(depth_value, lightest, height_scale, logz)
if color_height.nil? || depth_height.nil?
logz.info("nil height for: #{vertex}, #{color_height}, #{depth_height}") if logz
end
height = (carrier_weight * depth_height) + (detail_weight * color_height)
logz.info("height: #{height}, color_height: #{color_height}, depth_height: #{depth_height}") if logz
position = vertex.position
position.z = height
vector = vertex.position.vector_to(position)
xforms << vector
end
return xforms
end # calculate_vertz_height_by_light
def self.create_3d_depth_map(selection, image, depth_map, tw,
height_scale, carrier_weight, detail_weight, logz=nil)
logz.info("BEGIN create_3d_depth_map") if logz
entz = selection.entities
image_vert_light_map, image_light_counts = calculate_lightness_scores(selection, image, tw, "image", logz)
depth_vert_light_map, depth_light_counts = calculate_lightness_scores(selection, depth_map, tw, "depth", logz)
image_vertex_xforms = calculate_vertz_height_by_light(image_vert_light_map, depth_vert_light_map, height_scale, carrier_weight, detail_weight, logz)
vertz = image_vert_light_map.keys
entz.transform_by_vectors(vertz, image_vertex_xforms)
end # create_3d_depth_map
def self.main()
print("main")
model = Sketchup.active_model
logger = Logger.new("/Users/jjones/src/sketchup/ruby_scripts/log.txt")
texture_file_name = "/Users/jjones/Documents/texture.png"
depth_map_file = "/Users/jjones/Documents/midas-depth.png"
tw=Sketchup.create_texture_writer
cell_size = 4
image = load_texture_png(texture_file_name)
depth_map = load_texture_png(depth_map_file)
image_height = image.height
image_width = image.width
model.active_view.refresh
model.start_operation "create gridded texture"
print("create gridded texture")
face, fgroup = create_textured_face(model, image_width, image_height, texture_file_name)
create_grid_textured_face(face, fgroup, image_height, image_width, cell_size, tw, logz=logger)
# select the face group
model.selection.add(fgroup)
model.commit_operation
model.start_operation "create 3d depth map"
print("create 3d depth map")
model.active_view.refresh
selection = model.selection
print("selection: #{selection}, #{selection.class}")
return unless selection.any?
group = selection[0]
height_scale = 300
carrier_weight = 0.975
detail_weight = 0.025
create_3d_depth_map(group, image, depth_map, tw, height_scale, carrier_weight, detail_weight, logz=logger)
model.commit_operation
model.start_operation "smooth edges"
model.active_view.refresh
selection = model.selection
print("selection: #{selection}, #{selection.class}")
return unless selection.any?
group = selection[0]
smooth_edges(group)
model.commit_operation
model.start_operation "reapply texture"
# todo
model.commit_operation
logger.close
end # main
end # module
JJones.main
print("All done!")