This can be accomplished with a convolution of the distance transform.
Use a distance transform on the edge of the mask. Then threshold this distance transform to remove values beyond some distance. I think the secret to getting the shading is to convolve the distance transform result with a kernel that looks something like this:
[ -1.0 -1.0 -1.0
-1.0 0.0 0.0
-1.0 0.0 1.0 ]
This should get you started in the right direction:
#include "opencv/cv.h"
#include "opencv/highgui.h"
using namespace cv;
using namespace std;
int main() {
Mat mask, dist, bevel;
mask = Mat::zeros(200, 400, CV_8U);
rectangle(mask, Point(30,30), Point(180,180), Scalar(255), -1);
circle(mask, Point(30,30), 50, Scalar(0), -1);
circle(mask, Point(180,180), 50, Scalar(0), -1);
circle(mask, Point(300,100), 75, Scalar(255), -1);
imshow("1",mask);
//find edges and invert image for distance transform
Canny(mask, dist, 50, 150);
dist = 255-dist;
distanceTransform(dist, dist, CV_DIST_L2, CV_DIST_MASK_5);
threshold(dist, dist, 20, 20, CV_THRESH_TRUNC);
blur(dist, dist, Size(3,3));
dist.convertTo(bevel, CV_8U);
equalizeHist(bevel, bevel);
imshow("2",bevel);
//convolve with secret sauce
float d[] = {-1,-2,-3,
-2, 0, 0,
-3, 0, 1 };
Mat kernel(3, 3, CV_32F, d);
kernel = kernel - mean(kernel)[0];
filter2D(dist, dist, CV_32F, kernel);
//normalize filtering result to [-1, 1]
double maxVal;
minMaxLoc(dist, NULL, &maxVal);
dist = 128 * dist / maxVal;
//convert and display result
dist.convertTo(bevel, CV_8U, 1, 128);
bevel = bevel.mul(mask)/255;
imshow("3", bevel);
waitKey(0);
}