This prototype series focusses on realtime webcam pixel tracking. Pixel color information is used to steer simple 3D objects.
This prototype series focusses on realtime webcam pixel tracking. Pixel color information is used to steer simple 3D objects.
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Webcam Voxel Matrix</title>
<style>
body {
margin: 0;
overflow: hidden;
font-family: 'Inter', sans-serif;
background: #111;
color: #eee;
}
canvas {
display: block;
}
.ui-container {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
pointer-events: none;
text-align: center;
z-index: 10;
background: rgba(17, 17, 17, 0.9);
transition: opacity 1s ease-in-out;
}
.ui-container.controls {
top: 20px;
left: 20px;
width: auto;
height: auto;
padding: 20px;
background: rgba(17, 17, 17, 0.7);
border-radius: 12px;
flex-direction: column;
align-items: flex-start;
}
.hidden {
opacity: 0;
pointer-events: none;
}
button {
pointer-events: auto;
background-color: #4CAF50;
border: none;
color: white;
padding: 15px 32px;
text-align: center;
text-decoration: none;
display: inline-block;
font-size: 16px;
margin-top: 20px;
cursor: pointer;
border-radius: 8px;
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
transition: all 0.3s ease;
}
button:hover {
background-color: #45a049;
transform: translateY(-2px);
box-shadow: 0 6px 8px rgba(0,0,0,0.2);
}
h1 {
font-size: 2.5rem;
margin-bottom: 0.5rem;
}
p {
font-size: 1.2rem;
max-width: 80%;
}
#webcam-video, #webcam-canvas {
display: none;
}
.slider-group {
display: flex;
flex-direction: column;
align-items: flex-start;
margin-bottom: 15px;
pointer-events: auto;
}
.slider-group label {
margin-bottom: 5px;
font-size: 1rem;
text-shadow: 1px 1px 2px #000;
}
.slider-group input[type="range"] {
width: 200px;
-webkit-appearance: none;
background: #555;
border-radius: 5px;
outline: none;
opacity: 0.7;
transition: opacity .2s;
}
.slider-group input[type="range"]:hover {
opacity: 1;
}
.slider-group input[type="range"]::-webkit-slider-thumb {
-webkit-appearance: none;
appearance: none;
width: 15px;
height: 15px;
background: #4CAF50;
border-radius: 50%;
cursor: pointer;
}
</style>
</head>
<body>
<div id="ui-overlay" class="ui-container">
<h1>Webcam Voxel Matrix</h1>
<p>This app visualizes your webcam feed as a 3D grid of cubes. Click 'Start Webcam' and grant camera permission to begin.</p>
<button id="start-webcam-btn">Start Webcam</button>
</div>
<div id="control-panel" class="ui-container controls hidden">
<h2>Controls</h2>
<div class="slider-group">
<label for="scale-amount-slider">Scale Amount</label>
<input type="range" id="scale-amount-slider" min="0.1" max="5.0" step="0.1" value="1.5">
</div>
<div class="slider-group">
<label for="z-offset-slider">Z-Offset</label>
<input type="range" id="z-offset-slider" min="0.0" max="40.0" step="0.1" value="20.0">
</div>
<div class="slider-group">
<label for="position-smoothing-slider">Position Smoothing</label>
<input type="range" id="position-smoothing-slider" min="0.01" max="0.5" step="0.01" value="0.05">
</div>
<div class="slider-group">
<label for="rotation-smoothing-slider">Rotation Smoothing</label>
<input type="range" id="rotation-smoothing-slider" min="0.01" max="0.5" step="0.01" value="0.02">
</div>
</div>
<video id="webcam-video" autoplay playsinline></video>
<canvas id="webcam-canvas"></canvas>
<!-- Three.js Library -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
<!-- OrbitControls for camera interaction -->
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
<script>
// Global variables for Three.js
let scene, camera, renderer, controls;
let cubes = [];
const CUBE_COUNT_X = 128;
const CUBE_COUNT_Y = 64;
const CUBE_SPACING = 1;
// Webcam and canvas variables
let video, webcamCanvas, webcamContext;
// UI Control variables
let scaleAmountMultiplier = 1.5;
let zOffsetMultiplier = 20.0;
let positionSmoothingFactor = 0.05;
let rotationSmoothingFactor = 0.02;
// Smoothing variables
let smoothedData = [];
// Initialize the 3D scene
function initThreeJS() {
// Scene
scene = new THREE.Scene();
// Camera
camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);
camera.position.z = 100;
// Renderer
renderer = new THREE.WebGLRenderer({ antialias: true });
renderer.setSize(window.innerWidth, window.innerHeight);
renderer.setClearColor(0x111111, 1);
document.body.appendChild(renderer.domElement);
// Controls
controls = new THREE.OrbitControls(camera, renderer.domElement);
controls.enableDamping = true;
controls.dampingFactor = 0.25;
controls.enableZoom = true;
// Lighting
const ambientLight = new THREE.AmbientLight(0xffffff, 0.5);
scene.add(ambientLight);
const pointLight = new THREE.PointLight(0xffffff, 0.5);
pointLight.position.set(0, 50, 50);
scene.add(pointLight);
// Create the cubes
const geometry = new THREE.BoxGeometry(CUBE_SPACING, CUBE_SPACING, CUBE_SPACING);
const material = new THREE.MeshNormalMaterial();
const totalWidth = CUBE_COUNT_X * CUBE_SPACING;
const totalHeight = CUBE_COUNT_Y * CUBE_SPACING;
for (let y = 0; y < CUBE_COUNT_Y; y++) {
for (let x = 0; x < CUBE_COUNT_X; x++) {
const cube = new THREE.Mesh(geometry, material);
// Center the grid and flip the Y-axis
cube.position.x = x * CUBE_SPACING - totalWidth / 2 + CUBE_SPACING / 2;
cube.position.y = (CUBE_COUNT_Y - 1 - y) * CUBE_SPACING - totalHeight / 2 + CUBE_SPACING / 2;
scene.add(cube);
cubes.push(cube);
// Initialize smoothed data for each cube
smoothedData.push({
scale: new THREE.Vector3(1, 1, 1),
rotation: new THREE.Vector3(0, 0, 0)
});
}
}
camera.position.z = Math.max(totalWidth, totalHeight) * 1.5;
}
// Initialize webcam
async function initWebcam() {
video = document.getElementById('webcam-video');
webcamCanvas = document.getElementById('webcam-canvas');
webcamCanvas.width = CUBE_COUNT_X;
webcamCanvas.height = CUBE_COUNT_Y;
webcamContext = webcamCanvas.getContext('2d', { willReadFrequently: true });
try {
const stream = await navigator.mediaDevices.getUserMedia({ video: { width: CUBE_COUNT_X, height: CUBE_COUNT_Y } });
video.srcObject = stream;
video.onloadedmetadata = () => {
video.play();
document.getElementById('ui-overlay').classList.add('hidden');
document.getElementById('control-panel').classList.remove('hidden');
animate(); // Start the animation loop once video is ready
};
} catch (err) {
console.error("Error accessing the webcam: ", err);
const overlay = document.getElementById('ui-overlay');
overlay.innerHTML = `
<h1>Error</h1>
<p>Could not access webcam. Please ensure you have a camera connected and grant permission.</p>
`;
}
}
// Animation loop
function animate() {
requestAnimationFrame(animate);
// Update controls
controls.update();
if (video && video.readyState === video.HAVE_ENOUGH_DATA) {
// Draw video frame to the hidden canvas
webcamContext.drawImage(video, 0, 0, CUBE_COUNT_X, CUBE_COUNT_Y);
const imageData = webcamContext.getImageData(0, 0, CUBE_COUNT_X, CUBE_COUNT_Y).data;
// Update cube properties based on pixel data
for (let i = 0; i < cubes.length; i++) {
const cube = cubes[i];
const pixelIndex = i * 4;
const r = imageData[pixelIndex];
const g = imageData[pixelIndex + 1];
const b = imageData[pixelIndex + 2];
// Brightness (average of RGB values)
const brightness = (r + g + b) / 3;
// Map brightness to cube size and Z-position
const normalizedBrightness = brightness / 255;
const newScale = normalizedBrightness * scaleAmountMultiplier + 0.1;
const newZPosition = (normalizedBrightness - 0.5) * zOffsetMultiplier;
// Apply smoothing to scale and Z-position
smoothedData[i].scale.x += (newScale - smoothedData[i].scale.x) * positionSmoothingFactor;
smoothedData[i].scale.y += (newScale - smoothedData[i].scale.y) * positionSmoothingFactor;
smoothedData[i].scale.z += (newScale - smoothedData[i].scale.z) * positionSmoothingFactor;
cube.scale.copy(smoothedData[i].scale);
cube.position.z += (newZPosition - cube.position.z) * positionSmoothingFactor;
// Map color channels to rotation (scaled from 0 to 2*PI)
const newRotationX = (r / 255) * Math.PI * 2;
const newRotationY = (g / 255) * Math.PI * 2;
const newRotationZ = (b / 255) * Math.PI * 2;
// Apply smoothing twice to rotation
smoothedData[i].rotation.x += (newRotationX - smoothedData[i].rotation.x) * rotationSmoothingFactor;
smoothedData[i].rotation.y += (newRotationY - smoothedData[i].rotation.y) * rotationSmoothingFactor;
smoothedData[i].rotation.z += (newRotationZ - smoothedData[i].rotation.z) * rotationSmoothingFactor;
// Second smoothing pass
cube.rotation.x += (smoothedData[i].rotation.x - cube.rotation.x) * rotationSmoothingFactor;
cube.rotation.y += (smoothedData[i].rotation.y - cube.rotation.y) * rotationSmoothingFactor;
cube.rotation.z += (smoothedData[i].rotation.z - cube.rotation.z) * rotationSmoothingFactor;
}
}
renderer.render(scene, camera);
}
// Handle window resizing
window.addEventListener('resize', () => {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize(window.innerWidth, window.innerHeight);
}, false);
// Event listeners for UI controls
document.getElementById('start-webcam-btn').addEventListener('click', () => {
initThreeJS();
initWebcam();
});
document.getElementById('scale-amount-slider').addEventListener('input', (e) => {
scaleAmountMultiplier = parseFloat(e.target.value);
});
document.getElementById('z-offset-slider').addEventListener('input', (e) => {
zOffsetMultiplier = parseFloat(e.target.value);
});
document.getElementById('position-smoothing-slider').addEventListener('input', (e) => {
positionSmoothingFactor = parseFloat(e.target.value);
});
document.getElementById('rotation-smoothing-slider').addEventListener('input', (e) => {
rotationSmoothingFactor = parseFloat(e.target.value);
});
</script>
</body>
</html>
FULL CODE ONLINE: https://codepen.io/Tristan-Schulze/pen/YPyMyEv
COMPUTE SHADER UPGRADE
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Webcam Voxel Matrix</title>
<style>
body {
margin: 0;
overflow: hidden;
font-family: 'Inter', sans-serif;
background: #111;
color: #eee;
}
canvas {
display: block;
}
.ui-container {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
pointer-events: none;
text-align: center;
z-index: 10;
background: rgba(17, 17, 17, 0.9);
transition: opacity 1s ease-in-out;
}
.ui-container.controls {
top: 20px;
left: 20px;
width: auto;
height: auto;
padding: 20px;
background: rgba(17, 17, 17, 0.7);
border-radius: 12px;
flex-direction: column;
align-items: flex-start;
}
.hidden {
opacity: 0;
pointer-events: none;
}
button {
pointer-events: auto;
background-color: #4CAF50;
border: none;
color: white;
padding: 15px 32px;
text-align: center;
text-decoration: none;
display: inline-block;
font-size: 16px;
margin-top: 20px;
cursor: pointer;
border-radius: 8px;
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
transition: all 0.3s ease;
}
button:hover {
background-color: #45a049;
transform: translateY(-2px);
box-shadow: 0 6px 8px rgba(0,0,0,0.2);
}
h1 {
font-size: 2.5rem;
margin-bottom: 0.5rem;
}
p {
font-size: 1.2rem;
max-width: 80%;
}
#webcam-video, #webcam-canvas {
display: none;
}
.slider-group {
display: flex;
flex-direction: column;
align-items: flex-start;
margin-bottom: 15px;
pointer-events: auto;
}
.slider-group label {
margin-bottom: 5px;
font-size: 1rem;
text-shadow: 1px 1px 2px #000;
}
.slider-group input[type="range"] {
width: 200px;
-webkit-appearance: none;
background: #555;
border-radius: 5px;
outline: none;
opacity: 0.7;
transition: opacity .2s;
}
.slider-group input[type="range"]:hover {
opacity: 1;
}
.slider-group input[type="range"]::-webkit-slider-thumb {
-webkit-appearance: none;
appearance: none;
width: 15px;
height: 15px;
background: #4CAF50;
border-radius: 50%;
cursor: pointer;
}
</style>
</head>
<body>
<!-- Overlay for initial instructions -->
<div id="ui-overlay" class="ui-container">
<h1>Webcam Voxel Matrix</h1>
<p>This app visualizes your webcam feed as a 3D grid of cubes. Click 'Start Webcam' and grant camera permission to begin.</p>
<button id="start-webcam-btn">Start Webcam</button>
</div>
<!-- Control panel for sliders -->
<div id="control-panel" class="ui-container controls hidden">
<h2>Controls</h2>
<div class="slider-group">
<label for="scale-amount-slider">Scale Amount</label>
<input type="range" id="scale-amount-slider" min="0.1" max="5.0" step="0.1" value="1.5">
</div>
<div class="slider-group">
<label for="z-offset-slider">Z-Offset</label>
<input type="range" id="z-offset-slider" min="0.0" max="40.0" step="0.1" value="20.0">
</div>
<div class="slider-group">
<label for="position-smoothing-slider">Position Smoothing</label>
<input type="range" id="position-smoothing-slider" min="0.01" max="0.5" step="0.01" value="0.05">
</div>
<div class="slider-group">
<label for="rotation-smoothing-slider">Rotation Smoothing</label>
<input type="range" id="rotation-smoothing-slider" min="0.01" max="0.5" step="0.01" value="0.02">
</div>
<div class="slider-group">
<label for="light-intensity-slider">Light Intensity</label>
<input type="range" id="light-intensity-slider" min="0.1" max="2.0" step="0.1" value="1.0">
</div>
<div class="slider-group">
<label for="light-direction-slider">Light Direction</label>
<input type="range" id="light-direction-slider" min="0" max="360" step="1" value="45">
</div>
</div>
<!-- Hidden video and canvas for webcam feed processing -->
<video id="webcam-video" autoplay playsinline></video>
<canvas id="webcam-canvas"></canvas>
<!-- Three.js Library -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
<!-- OrbitControls for camera interaction -->
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
<script>
// Global variables for Three.js scene, camera, and renderer
let scene, camera, renderer, controls;
let instancedMesh;
let dummy = new THREE.Object3D();
// Webcam and canvas variables for video frame processing
let video, webcamCanvas, webcamContext;
// Voxel grid resolution and number of instances
const CUBE_COUNT_X = 256;
const CUBE_COUNT_Y = 128;
const NUM_INSTANCES = CUBE_COUNT_X * CUBE_COUNT_Y;
const CUBE_SPACING = 1.0;
// UI Control variables, initialized with default values
let scaleAmountMultiplier = 1.5;
let zOffsetMultiplier = 20.0;
let positionSmoothingFactor = 0.05;
let rotationSmoothingFactor = 0.02;
let lightIntensity = 1.0;
let lightDirectionAngle = 45;
// Array to store smoothed data for each cube
let smoothedData = [];
for(let i = 0; i < NUM_INSTANCES; i++) {
smoothedData.push({
scale: new THREE.Vector3(1, 1, 1),
rotation: new THREE.Euler(0, 0, 0),
zPos: 0
});
}
// Lighting
let ambientLight;
let directionalLight;
// Function to create and position the cubes in the scene
function createAndPositionCubes() {
// Remove previous instanced mesh if it exists to prevent memory leaks
if (instancedMesh) {
scene.remove(instancedMesh);
instancedMesh.geometry.dispose();
instancedMesh.material.dispose();
}
// Create the cubes using InstancedMesh for high performance
const geometry = new THREE.BoxGeometry(1, 1, 1);
const material = new THREE.MeshLambertMaterial({ color: 0xffffff });
instancedMesh = new THREE.InstancedMesh(geometry, material, NUM_INSTANCES);
scene.add(instancedMesh);
const totalWidth = CUBE_COUNT_X * CUBE_SPACING;
const totalHeight = CUBE_COUNT_Y * CUBE_SPACING;
// Loop through the grid and set the initial position for each cube
for (let y = 0; y < CUBE_COUNT_Y; y++) {
for (let x = 0; x < CUBE_COUNT_X; x++) {
const i = y * CUBE_COUNT_X + x;
dummy.position.x = x * CUBE_SPACING - totalWidth / 2 + CUBE_SPACING / 2;
dummy.position.y = (CUBE_COUNT_Y - 1 - y) * CUBE_SPACING - totalHeight / 2 + CUBE_SPACING / 2;
dummy.position.z = 0;
dummy.rotation.set(0, 0, 0);
dummy.scale.set(1, 1, 1);
dummy.updateMatrix();
instancedMesh.setMatrixAt(i, dummy.matrix);
}
}
instancedMesh.instanceMatrix.needsUpdate = true;
camera.position.z = Math.max(totalWidth, totalHeight) * 1.5;
camera.updateProjectionMatrix();
}
// Initialize the main Three.js scene and controls
function initThreeJS() {
// Scene
scene = new THREE.Scene();
// Camera
camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);
// Renderer
if (!renderer) {
renderer = new THREE.WebGLRenderer({ antialias: true });
document.body.appendChild(renderer.domElement);
}
renderer.setSize(window.innerWidth, window.innerHeight);
renderer.setClearColor(0x111111, 1);
// Controls
if (!controls) {
controls = new THREE.OrbitControls(camera, renderer.domElement);
controls.enableDamping = true;
controls.dampingFactor = 0.25;
controls.enableZoom = true;
}
// Lighting
ambientLight = new THREE.AmbientLight(0xffffff, 0.2);
scene.add(ambientLight);
directionalLight = new THREE.DirectionalLight(0xffffff, lightIntensity);
directionalLight.position.set(100, 100, 100);
scene.add(directionalLight);
// Create and position cubes initially
createAndPositionCubes();
}
// Initialize the webcam and start the video stream
async function initWebcam() {
video = document.getElementById('webcam-video');
webcamCanvas = document.getElementById('webcam-canvas');
webcamCanvas.width = CUBE_COUNT_X;
webcamCanvas.height = CUBE_COUNT_Y;
webcamContext = webcamCanvas.getContext('2d', { willReadFrequently: true });
try {
const stream = await navigator.mediaDevices.getUserMedia({ video: { width: CUBE_COUNT_X, height: CUBE_COUNT_Y } });
video.srcObject = stream;
video.onloadedmetadata = () => {
video.play();
// Hide the UI overlay and show the controls once the webcam is ready
document.getElementById('ui-overlay').classList.add('hidden');
document.getElementById('control-panel').classList.remove('hidden');
animate(); // Start the animation loop
};
} catch (err) {
console.error("Error accessing the webcam: ", err);
const overlay = document.getElementById('ui-overlay');
overlay.innerHTML = `
<h1>Error</h1>
<p>Could not access webcam. Please ensure you have a camera connected and grant permission.</p>
`;
}
}
// The main animation loop for the scene
function animate() {
requestAnimationFrame(animate);
// Update OrbitControls
controls.update();
// Update directional light properties from sliders
directionalLight.intensity = lightIntensity;
const angle = lightDirectionAngle * Math.PI / 180;
directionalLight.position.x = Math.sin(angle) * 150;
directionalLight.position.z = Math.cos(angle) * 150;
if (video && video.readyState === video.HAVE_ENOUGH_DATA) {
// Draw a frame from the video to the hidden 2D canvas
webcamContext.drawImage(video, 0, 0, CUBE_COUNT_X, CUBE_COUNT_Y);
const imageData = webcamContext.getImageData(0, 0, CUBE_COUNT_X, CUBE_COUNT_Y).data;
// Process each pixel and update the corresponding cube
for (let y = 0; y < CUBE_COUNT_Y; y++) {
for (let x = 0; x < CUBE_COUNT_X; x++) {
const i = y * CUBE_COUNT_X + x;
const pixelIndex = i * 4;
const r = imageData[pixelIndex];
const g = imageData[pixelIndex + 1];
const b = imageData[pixelIndex + 2];
// Calculate brightness (average of RGB)
const brightness = (r + g + b) / 3;
// Map brightness to cube size and Z-position
const normalizedBrightness = brightness / 255;
const newScale = normalizedBrightness * scaleAmountMultiplier + 0.1;
const newZPosition = (normalizedBrightness - 0.5) * zOffsetMultiplier;
// Apply smoothing to the new values
smoothedData[i].scale.x += (newScale - smoothedData[i].scale.x) * positionSmoothingFactor;
smoothedData[i].scale.y += (newScale - smoothedData[i].scale.y) * positionSmoothingFactor;
smoothedData[i].scale.z += (newScale - smoothedData[i].scale.z) * positionSmoothingFactor;
smoothedData[i].zPos += (newZPosition - smoothedData[i].zPos) * positionSmoothingFactor;
// Map color channels to rotation angles
const newRotationX = (r / 255) * Math.PI * 2;
const newRotationY = (g / 255) * Math.PI * 2;
const newRotationZ = (b / 255) * Math.PI * 2;
// Apply smoothing twice for a more gradual rotation change
smoothedData[i].rotation.x += (newRotationX - smoothedData[i].rotation.x) * rotationSmoothingFactor;
smoothedData[i].rotation.y += (newRotationY - smoothedData[i].rotation.y) * rotationSmoothingFactor;
smoothedData[i].rotation.z += (newRotationZ - smoothedData[i].rotation.z) * rotationSmoothingFactor;
// Update the dummy object with new values
dummy.position.x = x * CUBE_SPACING - (CUBE_COUNT_X * CUBE_SPACING) / 2 + CUBE_SPACING / 2;
dummy.position.y = (CUBE_COUNT_Y - 1 - y) * CUBE_SPACING - (CUBE_COUNT_Y * CUBE_SPACING) / 2 + CUBE_SPACING / 2;
dummy.position.z = smoothedData[i].zPos;
dummy.scale.copy(smoothedData[i].scale);
dummy.rotation.set(
smoothedData[i].rotation.x + (newRotationX - smoothedData[i].rotation.x) * rotationSmoothingFactor,
smoothedData[i].rotation.y + (newRotationY - smoothedData[i].rotation.y) * rotationSmoothingFactor,
smoothedData[i].rotation.z + (newRotationZ - smoothedData[i].rotation.z) * rotationSmoothingFactor
);
dummy.updateMatrix();
instancedMesh.setMatrixAt(i, dummy.matrix);
}
}
// Tell Three.js to update the instanced mesh with the new matrices
instancedMesh.instanceMatrix.needsUpdate = true;
}
// Render the scene
renderer.render(scene, camera);
}
// Handle window resizing
window.addEventListener('resize', () => {
if (camera) {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize(window.innerWidth, window.innerHeight);
}
}, false);
// Event listener for the "Start Webcam" button
document.getElementById('start-webcam-btn').addEventListener('click', () => {
initThreeJS();
initWebcam();
});
// Event listeners for the UI sliders
document.getElementById('scale-amount-slider').addEventListener('input', (e) => {
scaleAmountMultiplier = parseFloat(e.target.value);
});
document.getElementById('z-offset-slider').addEventListener('input', (e) => {
zOffsetMultiplier = parseFloat(e.target.value);
});
document.getElementById('position-smoothing-slider').addEventListener('input', (e) => {
positionSmoothingFactor = parseFloat(e.target.value);
});
document.getElementById('rotation-smoothing-slider').addEventListener('input', (e) => {
rotationSmoothingFactor = parseFloat(e.target.value);
});
document.getElementById('light-intensity-slider').addEventListener('input', (e) => {
lightIntensity = parseFloat(e.target.value);
});
document.getElementById('light-direction-slider').addEventListener('input', (e) => {
lightDirectionAngle = parseFloat(e.target.value);
});
</script>
</body>
</html>
COMPUTE SHADING + FOG + BLOOM
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Webcam Voxel Matrix</title>
<style>
body {
margin: 0;
overflow: hidden;
font-family: 'Inter', sans-serif;
background: #111;
color: #eee;
}
canvas {
display: block;
}
.ui-container {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
pointer-events: none;
text-align: center;
z-index: 10;
background: rgba(17, 17, 17, 0.9);
transition: opacity 1s ease-in-out;
}
.ui-container.controls {
top: 20px;
left: 20px;
width: auto;
height: auto;
padding: 20px;
background: rgba(17, 17, 17, 0.7);
border-radius: 12px;
flex-direction: column;
align-items: flex-start;
}
.hidden {
opacity: 0;
pointer-events: none;
}
.ui-button {
pointer-events: auto;
background-color: #4CAF50;
border: none;
color: white;
padding: 15px 32px;
text-align: center;
text-decoration: none;
display: inline-block;
font-size: 16px;
margin-top: 20px;
cursor: pointer;
border-radius: 8px;
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
transition: all 0.3s ease;
}
.ui-button:hover {
background-color: #45a049;
transform: translateY(-2px);
box-shadow: 0 6px 8px rgba(0,0,0,0.2);
}
h1 {
font-size: 2.5rem;
margin-bottom: 0.5rem;
}
p {
font-size: 1.2rem;
max-width: 80%;
}
#webcam-video, #webcam-canvas {
display: none;
}
.slider-group {
display: flex;
flex-direction: column;
align-items: flex-start;
margin-bottom: 15px;
pointer-events: auto;
}
.slider-group label {
margin-bottom: 5px;
font-size: 1rem;
text-shadow: 1px 1px 2px #000;
}
.slider-group input[type="range"] {
width: 200px;
-webkit-appearance: none;
background: #555;
border-radius: 5px;
outline: none;
opacity: 0.7;
transition: opacity .2s;
}
.slider-group input[type="range"]:hover {
opacity: 1;
}
.slider-group input[type="range"]::-webkit-slider-thumb {
-webkit-appearance: none;
appearance: none;
width: 15px;
height: 15px;
background: #4CAF50;
border-radius: 50%;
cursor: pointer;
}
.controls-row {
display: flex;
gap: 15px;
margin-top: 15px;
}
</style>
</head>
<body>
<!-- Overlay for initial instructions -->
<div id="ui-overlay" class="ui-container">
<h1>Webcam Voxel Matrix</h1>
<p>This app visualizes your webcam feed as a 3D grid of cubes. Click 'Start Webcam' and grant camera permission to begin.</p>
<button id="start-webcam-btn" class="ui-button">Start Webcam</button>
</div>
<!-- Control panel for sliders -->
<div id="control-panel" class="ui-container controls hidden">
<h2>Controls</h2>
<div class="slider-group">
<label for="scale-amount-slider">Scale Amount</label>
<input type="range" id="scale-amount-slider" min="0.1" max="5.0" step="0.1" value="1.5">
</div>
<div class="slider-group">
<label for="z-offset-slider">Z-Offset</label>
<input type="range" id="z-offset-slider" min="0.0" max="80.0" step="0.1" value="20.0">
</div>
<div class="slider-group">
<label for="position-smoothing-slider">Position Smoothing</label>
<input type="range" id="position-smoothing-slider" min="0.01" max="0.5" step="0.01" value="0.05">
</div>
<div class="slider-group">
<label for="rotation-smoothing-slider">Rotation Smoothing</label>
<input type="range" id="rotation-smoothing-slider" min="0.01" max="0.5" step="0.01" value="0.02">
</div>
<div class="slider-group">
<label for="light-intensity-slider">Light Intensity</label>
<input type="range" id="light-intensity-slider" min="0.1" max="2.0" step="0.1" value="1.0">
</div>
<div class="slider-group">
<label for="light-direction-slider">Light Direction</label>
<input type="range" id="light-direction-slider" min="0" max="360" step="1" value="45">
</div>
<div class="controls-row">
<button id="toggle-bloom-btn" class="ui-button">Disable Bloom</button>
<button id="fullscreen-btn" class="ui-button">Go Fullscreen</button>
</div>
<div id="bloom-controls">
<div class="slider-group">
<label for="bloom-strength-slider">Bloom Strength</label>
<input type="range" id="bloom-strength-slider" min="0.0" max="3.0" step="0.1" value="1.0">
</div>
<div class="slider-group">
<label for="bloom-threshold-slider">Bloom Threshold</label>
<input type="range" id="bloom-threshold-slider" min="0.0" max="1.0" step="0.01" value="0.5">
</div>
<div class="slider-group">
<label for="bloom-radius-slider">Bloom Radius</label>
<input type="range" id="bloom-radius-slider" min="0.0" max="1.0" step="0.01" value="0.4">
</div>
</div>
<div id="fog-controls">
<div class="slider-group">
<label for="fog-near-slider">Fog Near</label>
<input type="range" id="fog-near-slider" min="1" max="200" step="1" value="100">
</div>
<div class="slider-group">
<label for="fog-far-slider">Fog Far</label>
<input type="range" id="fog-far-slider" min="200" max="1000" step="1" value="300">
</div>
</div>
</div>
<!-- Hidden video and canvas for webcam feed processing -->
<video id="webcam-video" autoplay playsinline></video>
<canvas id="webcam-canvas"></canvas>
<!-- Three.js Library -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
<!-- OrbitControls for camera interaction -->
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/controls/OrbitControls.js"></script>
<!-- Post-processing shaders -->
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/shaders/CopyShader.js"></script>
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/shaders/LuminosityHighPassShader.js"></script>
<!-- Post-processing passes -->
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/EffectComposer.js"></script>
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/RenderPass.js"></script>
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/ShaderPass.js"></script>
<script src="https://cdn.jsdelivr.net/npm/three@0.128.0/examples/js/postprocessing/UnrealBloomPass.js"></script>
<script>
// Global variables for Three.js scene, camera, and renderer
let scene, camera, renderer, controls, composer;
let instancedMesh;
let dummy = new THREE.Object3D();
// Webcam and canvas variables for video frame processing
let video, webcamCanvas, webcamContext;
// Post-processing passes
let renderPass, unrealBloomPass;
let isBloomEnabled = true;
// Voxel grid resolution and number of instances
const CUBE_COUNT_X = 400;
const CUBE_COUNT_Y = 300;
const NUM_INSTANCES = CUBE_COUNT_X * CUBE_COUNT_Y;
const CUBE_SPACING = 1.0;
// UI Control variables, initialized with default values
let scaleAmountMultiplier = 1.5;
let zOffsetMultiplier = 20.0;
let positionSmoothingFactor = 0.05;
let rotationSmoothingFactor = 0.02;
let lightIntensity = 1.0;
let lightDirectionAngle = 45;
let fogNear = 100;
let fogFar = 300;
// Array to store smoothed data for each cube
let smoothedData = [];
for(let i = 0; i < NUM_INSTANCES; i++) {
smoothedData.push({
scale: new THREE.Vector3(1, 1, 1),
rotation: new THREE.Euler(0, 0, 0),
zPos: 0
});
}
// Lighting
let ambientLight;
let directionalLight1;
// Function to create and position the cubes in the scene
function createAndPositionCubes() {
// Remove previous instanced mesh if it exists to prevent memory leaks
if (instancedMesh) {
scene.remove(instancedMesh);
instancedMesh.geometry.dispose();
instancedMesh.material.dispose();
}
// Create the cubes using InstancedMesh for high performance
const geometry = new THREE.BoxGeometry(1, 1, 1);
const material = new THREE.MeshLambertMaterial({ color: 0xffffff });
instancedMesh = new THREE.InstancedMesh(geometry, material, NUM_INSTANCES);
scene.add(instancedMesh);
const totalWidth = CUBE_COUNT_X * CUBE_SPACING;
const totalHeight = CUBE_COUNT_Y * CUBE_SPACING;
// Loop through the grid and set the initial position for each cube
for (let y = 0; y < CUBE_COUNT_Y; y++) {
for (let x = 0; x < CUBE_COUNT_X; x++) {
const i = y * CUBE_COUNT_X + x;
dummy.position.x = x * CUBE_SPACING - totalWidth / 2 + CUBE_SPACING / 2;
dummy.position.y = (CUBE_COUNT_Y - 1 - y) * CUBE_SPACING - totalHeight / 2 + CUBE_SPACING / 2;
dummy.position.z = 0;
dummy.rotation.set(0, 0, 0);
dummy.scale.set(1, 1, 1);
dummy.updateMatrix();
instancedMesh.setMatrixAt(i, dummy.matrix);
}
}
instancedMesh.instanceMatrix.needsUpdate = true;
camera.position.z = Math.max(totalWidth, totalHeight) * 1.5;
camera.updateProjectionMatrix();
}
// Initialize the main Three.js scene and controls
function initThreeJS() {
// Scene
scene = new THREE.Scene();
scene.fog = new THREE.Fog(0x111111, fogNear, fogFar);
// Camera
camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);
// Renderer
if (!renderer) {
renderer = new THREE.WebGLRenderer({ antialias: true });
document.body.appendChild(renderer.domElement);
}
renderer.setSize(window.innerWidth, window.innerHeight);
renderer.setClearColor(0x111111, 1);
// Controls
if (!controls) {
controls = new THREE.OrbitControls(camera, renderer.domElement);
controls.enableDamping = true;
controls.dampingFactor = 0.25;
controls.enableZoom = true;
}
// Lighting
ambientLight = new THREE.AmbientLight(0xffffff, 0); // Ambient light set to black
scene.add(ambientLight);
directionalLight1 = new THREE.DirectionalLight(0xffffff, lightIntensity);
directionalLight1.position.set(100, 100, 100);
scene.add(directionalLight1);
// Post-processing setup
composer = new THREE.EffectComposer(renderer);
renderPass = new THREE.RenderPass(scene, camera);
composer.addPass(renderPass);
// Initialize Bloom pass, starting with a default strength
unrealBloomPass = new THREE.UnrealBloomPass(new THREE.Vector2(window.innerWidth, window.innerHeight), 1.0, 0.4, 0.85);
composer.addPass(unrealBloomPass);
// Create and position cubes initially
createAndPositionCubes();
}
// Initialize the webcam and start the video stream
async function initWebcam() {
video = document.getElementById('webcam-video');
webcamCanvas = document.getElementById('webcam-canvas');
webcamCanvas.width = CUBE_COUNT_X;
webcamCanvas.height = CUBE_COUNT_Y;
webcamContext = webcamCanvas.getContext('2d', { willReadFrequently: true });
try {
const stream = await navigator.mediaDevices.getUserMedia({ video: { width: CUBE_COUNT_X, height: CUBE_COUNT_Y } });
video.srcObject = stream;
video.onloadedmetadata = () => {
video.play();
// Hide the UI overlay and show the controls once the webcam is ready
document.getElementById('ui-overlay').classList.add('hidden');
document.getElementById('control-panel').classList.remove('hidden');
animate(); // Start the animation loop
};
} catch (err) {
console.error("Error accessing the webcam: ", err);
const overlay = document.getElementById('ui-overlay');
overlay.innerHTML = `
<h1>Error</h1>
<p>Could not access webcam. Please ensure you have a camera connected and grant permission.</p>
`;
}
}
// The main animation loop for the scene
function animate() {
requestAnimationFrame(animate);
// Update OrbitControls
controls.update();
// Update directional light properties from sliders
directionalLight1.intensity = lightIntensity;
const angle1 = lightDirectionAngle * Math.PI / 180;
directionalLight1.position.x = Math.sin(angle1) * 150;
directionalLight1.position.z = Math.cos(angle1) * 150;
// Update fog properties
scene.fog.near = fogNear;
scene.fog.far = fogFar;
if (video && video.readyState === video.HAVE_ENOUGH_DATA) {
// Draw a frame from the video to the hidden 2D canvas
webcamContext.drawImage(video, 0, 0, CUBE_COUNT_X, CUBE_COUNT_Y);
const imageData = webcamContext.getImageData(0, 0, CUBE_COUNT_X, CUBE_COUNT_Y).data;
// Process each pixel and update the corresponding cube
for (let y = 0; y < CUBE_COUNT_Y; y++) {
for (let x = 0; x < CUBE_COUNT_X; x++) {
const i = y * CUBE_COUNT_X + x;
const pixelIndex = i * 4;
const r = imageData[pixelIndex];
const g = imageData[pixelIndex + 1];
const b = imageData[pixelIndex + 2];
// Calculate brightness (average of RGB)
const brightness = (r + g + b) / 3;
// Map brightness to cube size and Z-position
const normalizedBrightness = brightness / 255;
const newScale = normalizedBrightness * scaleAmountMultiplier + 0.1;
const newZPosition = (normalizedBrightness - 0.5) * zOffsetMultiplier;
// Apply smoothing to the new values
smoothedData[i].scale.x += (newScale - smoothedData[i].scale.x) * positionSmoothingFactor;
smoothedData[i].scale.y += (newScale - smoothedData[i].scale.y) * positionSmoothingFactor;
smoothedData[i].scale.z += (newScale - smoothedData[i].scale.z) * positionSmoothingFactor;
smoothedData[i].zPos += (newZPosition - smoothedData[i].zPos) * positionSmoothingFactor;
// Map color channels to rotation angles
const newRotationX = (r / 255) * Math.PI * 2;
const newRotationY = (g / 255) * Math.PI * 2;
const newRotationZ = (b / 255) * Math.PI * 2;
// Apply smoothing twice for a more gradual rotation change
smoothedData[i].rotation.x += (newRotationX - smoothedData[i].rotation.x) * rotationSmoothingFactor;
smoothedData[i].rotation.y += (newRotationY - smoothedData[i].rotation.y) * rotationSmoothingFactor;
smoothedData[i].rotation.z += (newRotationZ - smoothedData[i].rotation.z) * rotationSmoothingFactor;
// Update the dummy object with new values
dummy.position.x = x * CUBE_SPACING - (CUBE_COUNT_X * CUBE_SPACING) / 2 + CUBE_SPACING / 2;
dummy.position.y = (CUBE_COUNT_Y - 1 - y) * CUBE_SPACING - (CUBE_COUNT_Y * CUBE_SPACING) / 2 + CUBE_SPACING / 2;
dummy.position.z = smoothedData[i].zPos;
dummy.scale.copy(smoothedData[i].scale);
dummy.rotation.set(
smoothedData[i].rotation.x + (newRotationX - smoothedData[i].rotation.x) * rotationSmoothingFactor,
smoothedData[i].rotation.y + (newRotationY - smoothedData[i].rotation.y) * rotationSmoothingFactor,
smoothedData[i].rotation.z + (newRotationZ - smoothedData[i].rotation.z) * rotationSmoothingFactor
);
dummy.updateMatrix();
instancedMesh.setMatrixAt(i, dummy.matrix);
}
}
// Tell Three.js to update the instanced mesh with the new matrices
instancedMesh.instanceMatrix.needsUpdate = true;
}
// Render the scene using the composer for post-processing effects
composer.render();
}
// Handle window resizing
window.addEventListener('resize', () => {
if (camera && renderer && composer) {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize(window.innerWidth, window.innerHeight);
composer.setSize(window.innerWidth, window.innerHeight);
}
}, false);
// Event listener for the "Start Webcam" button
document.getElementById('start-webcam-btn').addEventListener('click', () => {
initThreeJS();
initWebcam();
});
// Event listeners for the UI sliders
document.getElementById('scale-amount-slider').addEventListener('input', (e) => {
scaleAmountMultiplier = parseFloat(e.target.value);
});
document.getElementById('z-offset-slider').addEventListener('input', (e) => {
zOffsetMultiplier = parseFloat(e.target.value);
});
document.getElementById('position-smoothing-slider').addEventListener('input', (e) => {
positionSmoothingFactor = parseFloat(e.target.value);
});
document.getElementById('rotation-smoothing-slider').addEventListener('input', (e) => {
rotationSmoothingFactor = parseFloat(e.target.value);
});
document.getElementById('light-intensity-slider').addEventListener('input', (e) => {
lightIntensity = parseFloat(e.target.value);
});
document.getElementById('light-direction-slider').addEventListener('input', (e) => {
lightDirectionAngle = parseFloat(e.target.value);
});
document.getElementById('fog-near-slider').addEventListener('input', (e) => {
fogNear = parseFloat(e.target.value);
});
document.getElementById('fog-far-slider').addEventListener('input', (e) => {
fogFar = parseFloat(e.target.value);
});
// Toggle button for bloom effect
document.getElementById('toggle-bloom-btn').addEventListener('click', () => {
isBloomEnabled = !isBloomEnabled;
const button = document.getElementById('toggle-bloom-btn');
if (isBloomEnabled) {
unrealBloomPass.strength = parseFloat(document.getElementById('bloom-strength-slider').value);
button.textContent = 'Disable Bloom';
} else {
unrealBloomPass.strength = 0;
button.textContent = 'Enable Bloom';
}
});
// Event listeners for Bloom sliders
document.getElementById('bloom-strength-slider').addEventListener('input', (e) => {
if (isBloomEnabled) {
unrealBloomPass.strength = parseFloat(e.target.value);
}
});
document.getElementById('bloom-threshold-slider').addEventListener('input', (e) => {
if (isBloomEnabled) {
unrealBloomPass.threshold = parseFloat(e.target.value);
}
});
document.getElementById('bloom-radius-slider').addEventListener('input', (e) => {
if (isBloomEnabled) {
unrealBloomPass.radius = parseFloat(e.target.value);
}
});
// Fullscreen button event listener
document.getElementById('fullscreen-btn').addEventListener('click', () => {
if (!document.fullscreenElement) {
document.documentElement.requestFullscreen().then(() => {
document.getElementById('fullscreen-btn').textContent = 'Exit Fullscreen';
}).catch(err => {
console.error(`Error attempting to enable full-screen mode: ${err.message} (${err.name})`);
});
} else {
document.exitFullscreen().then(() => {
document.getElementById('fullscreen-btn').textContent = 'Go Fullscreen';
});
}
});
</script>
</body>
</html>