diff --git a/commands/rm.go b/commands/rm.go
index 73144066..9e72af03 100644
--- a/commands/rm.go
+++ b/commands/rm.go
@@ -149,6 +149,11 @@ func rmAllInactive(ctx context.Context, txn *store.Txn, dockerCli command.Cli, i
 		builders[i] = &nginfo{ng: ng}
 	}
 
+	ctxbuilders, err := dockerCli.ContextStore().List()
+	if err != nil {
+		return err
+	}
+
 	eg, _ := errgroup.WithContext(ctx)
 	for _, b := range builders {
 		func(b *nginfo) {
@@ -156,6 +161,11 @@ func rmAllInactive(ctx context.Context, txn *store.Txn, dockerCli command.Cli, i
 				if err := loadNodeGroupData(ctx, dockerCli, b); err != nil {
 					return errors.Wrapf(err, "cannot load %s", b.ng.Name)
 				}
+				for _, cb := range ctxbuilders {
+					if b.ng.Driver == "docker" && len(b.ng.Nodes) == 1 && b.ng.Nodes[0].Endpoint == cb.Name {
+						return errors.Errorf("context builder cannot be removed, run `docker context rm %s` to remove this context", cb.Name)
+					}
+				}
 				if b.ng.Dynamic {
 					return nil
 				}