diff --git a/commands/rm.go b/commands/rm.go
index 73f33be9..6e387ed4 100644
--- a/commands/rm.go
+++ b/commands/rm.go
@@ -43,31 +43,37 @@ func runRm(dockerCli command.Cli, in rmOptions) error {
 		return rmAllInactive(ctx, txn, dockerCli, in)
 	}
 
+	var ng *store.NodeGroup
 	if in.builder != "" {
-		ng, err := storeutil.GetNodeGroup(txn, dockerCli, in.builder)
+		ng, err = storeutil.GetNodeGroup(txn, dockerCli, in.builder)
 		if err != nil {
 			return err
 		}
-		err1 := rm(ctx, dockerCli, in, ng)
-		if err := txn.Remove(ng.Name); err != nil {
+	} else {
+		ng, err = storeutil.GetCurrentInstance(txn, dockerCli)
+		if err != nil {
 			return err
 		}
-		return err1
+	}
+	if ng == nil {
+		return nil
 	}
 
-	ng, err := storeutil.GetCurrentInstance(txn, dockerCli)
+	ctxbuilders, err := dockerCli.ContextStore().List()
 	if err != nil {
 		return err
 	}
-	if ng != nil {
-		err1 := rm(ctx, dockerCli, in, ng)
-		if err := txn.Remove(ng.Name); err != nil {
-			return err
+	for _, cb := range ctxbuilders {
+		if ng.Driver == "docker" && len(ng.Nodes) == 1 && ng.Nodes[0].Endpoint == cb.Name {
+			return errors.Errorf("context builder cannot be removed, run `docker context rm %s` to remove this context", cb.Name)
 		}
-		return err1
 	}
 
-	return nil
+	err1 := rm(ctx, dockerCli, in, ng)
+	if err := txn.Remove(ng.Name); err != nil {
+		return err
+	}
+	return err1
 }
 
 func rmCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {