I am not able to reproduce this. The functions locks the group as expected in my test.
You say āblueā - the default style in SU display a red boundingbox when you select a locked instance. However this is style dependant. Please check the Entity Info.
If you still see it doesnāt working, please provide a full example that reproduce the issue. Including any models that might be needed.
Here is the test snipped I used:
// SLAPI-Test.cpp : Defines the entry point for the console application.
//
#include "stdafx.h"
#include <array>
#include <assert.h>
#include <iostream>
#include <vector>
#include "SketchUpAPI/sketchup.h"
int _tmain(int argc, _TCHAR* argv[])
{
SUInitialize();
SUModelRef model = SU_INVALID;
SUResult result = SUModelCreateFromFile(&model, "C:\\Users\\tthomas2\\SourceTree\\SLAPI-Test\\models\\SU-31417.skp");
assert(result == SU_ERROR_NONE);
SUEntitiesRef entities = SU_INVALID;
result = SUModelGetEntities(model, &entities);
assert(result == SU_ERROR_NONE);
size_t num_groups = 0;
result = SUEntitiesGetNumGroups(entities, &num_groups);
assert(result == SU_ERROR_NONE);
std::vector<SUGroupRef> groups(num_groups, SU_INVALID);
size_t num_group_out = 0;
result = SUEntitiesGetGroups(entities, num_groups, &groups[0], &num_group_out);
assert(result == SU_ERROR_NONE);
SUGroupRef group = groups.back();
SUEntitiesRef group_entities = SU_INVALID;
result = SUGroupGetEntities(group, &group_entities);
assert(result == SU_ERROR_NONE);
// SUGroupToComponentInstance
SUComponentInstanceRef instance = SU_INVALID;
instance = SUGroupToComponentInstance(group);
assert(SUIsValid(instance));
// SUGroupFromComponentInstance
SUGroupRef group2 = SUGroupFromComponentInstance(instance);
assert(SUIsValid(group2));
// SUComponentInstanceIsLocked
bool is_locked = false;
result = SUComponentInstanceIsLocked(instance, &is_locked);
assert(result == SU_ERROR_NONE);
assert(!is_locked);
// SUComponentInstanceSetLocked
result = SUComponentInstanceSetLocked(instance, true);
assert(result == SU_ERROR_NONE);
result = SUComponentInstanceIsLocked(instance, &is_locked);
assert(result == SU_ERROR_NONE);
assert(is_locked);
// SUComponentInstanceSaveAs
result = SUModelSaveToFile(model, "C:\\Users\\tthomas2\\SourceTree\\SLAPI-Test\\models\\SUComponentInstanceSetLocked.skp");
assert(result == SU_ERROR_NONE);
SUModelRelease(&model);
SUTerminate();
return 0;
}
The resulting file:

And the SKP file it reads in: SU-31417.skp (30.2 KB)